Skip to content

armgit / camel

Public Imported from GitHub
0

๐Ÿซ CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org

feat: Add Ubuntu docker runtime (#1952)

jjyaoao4f83f7cb1931
diff --git a/camel/runtime/__init__.py b/camel/runtime/__init__.py
index 024b7b066..3159a1c9d 100644
--- a/camel/runtime/__init__.py
+++ b/camel/runtime/__init__.py
@@ -16,6 +16,7 @@ from .configs import TaskConfig
from .docker_runtime import DockerRuntime
from .llm_guard_runtime import LLMGuardRuntime
from .remote_http_runtime import RemoteHttpRuntime
+from .ubuntu_docker_runtime import UbuntuDockerRuntime
# TODO: Add Celery Runtime to support distributed computing,
# Rate Limiting, Load Balancing, etc.
@@ -26,4 +27,5 @@ __all__ = [
"RemoteHttpRuntime",
"LLMGuardRuntime",
"TaskConfig",
+ "UbuntuDockerRuntime",
]
diff --git a/camel/runtime/ubuntu_docker_runtime.py b/camel/runtime/ubuntu_docker_runtime.py
new file mode 100644
index 000000000..f2149d583
--- /dev/null
+++ b/camel/runtime/ubuntu_docker_runtime.py
@@ -0,0 +1,340 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import logging
+import sys
+import time
+from pathlib import Path
+from typing import Callable, List, Optional, Union
+
+from camel.runtime.docker_runtime import DockerRuntime
+from camel.toolkits import FunctionTool
+
+logger = logging.getLogger(__name__)
+
+
+class UbuntuDockerRuntime(DockerRuntime):
+ r"""A specialized Docker runtime for Ubuntu-based environments.
+
+ This runtime includes specific configurations and setup for Ubuntu
+ containers, including proper Python path handling and environment setup.
+ It provides methods for executing Python files, managing the container
+ lifecycle, and handling file operations within the Ubuntu container.
+
+ Attributes:
+ python_path (str): Path to the Python interpreter in the container
+ docker_config (dict): Configuration dict for Docker container setup
+ """
+
+ def __init__(
+ self,
+ image: str,
+ port: int = 0,
+ remove: bool = True,
+ python_path: str = "/usr/bin/python3",
+ **kwargs,
+ ):
+ r"""Initialize the Ubuntu Docker Runtime.
+
+ Args:
+ image (str): Docker image name to use
+ port (int, optional): Port to expose. Defaults to 0 (random port)
+ remove (bool, optional): Whether to remove container after use.
+ Defaults to True
+ python_path (str, optional): Path to Python interpreter.
+ Defaults to "/usr/bin/python3"
+ **kwargs: Additional arguments passed to DockerRuntime
+ """
+ super().__init__(image=image, port=port, remove=remove, **kwargs)
+
+ self.python_path = python_path
+ logger.info(
+ f"Initializing UbuntuDockerRuntime with python_path: {python_path}"
+ )
+
+ # Set default environment variables for Ubuntu
+ self.docker_config.setdefault("environment", {})
+ self.docker_config["environment"].update(
+ {
+ "PYTHON_PATH": python_path,
+ "PYTHON_EXECUTABLE": python_path,
+ "PATH": "/usr/local/bin:/usr/bin:/bin",
+ "PYTHONUNBUFFERED": "1",
+ }
+ )
+ logger.info(
+ f"Environment variables set: {self.docker_config['environment']}"
+ )
+
+ # Add default working directory
+ self.docker_config.setdefault("working_dir", "/app")
+
+ # Setup default volume mounts
+ self._setup_default_mounts()
+
+ def add(
+ self,
+ funcs: Union[FunctionTool, List[FunctionTool]],
+ entrypoint: str,
+ redirect_stdout: bool = False,
+ arguments: Optional[dict] = None,
+ ) -> "UbuntuDockerRuntime":
+ r"""Add functions to the runtime with Ubuntu-specific modifications.
+
+ Args:
+ funcs: Function(s) to add to the runtime
+ entrypoint: Entry point for function execution
+ redirect_stdout: Whether to redirect stdout
+ arguments: Optional arguments for function execution
+
+ Returns:
+ Self for method chaining
+ """
+ if not isinstance(funcs, list):
+ funcs = [funcs]
+
+ # Modify the code execution command to use python3
+ for func in funcs:
+ logger.info(f"Processing function: {func.get_function_name()}")
+ if hasattr(func, 'command'):
+ logger.info(f"Original command: {func.command}")
+ if isinstance(func.command, list):
+ if 'python' in func.command:
+ idx = func.command.index('python')
+ func.command[idx] = self.python_path
+ logger.info(f"Modified command: {func.command}")
+ else:
+ logger.info(
+ f"No command attribute found for function "
+ f"{func.get_function_name()}"
+ )
+
+ super().add(funcs, entrypoint, redirect_stdout, arguments)
+ return self
+
+ def _setup_default_mounts(self):
+ r"""Setup default volume mounts for the container.
+
+ This method can be extended to add Ubuntu-specific volume mounts.
+ """
+ pass
+
+ def build(self, time_out: int = 15) -> "UbuntuDockerRuntime":
+ r"""Build and initialize the Ubuntu container with proper setup.
+
+ Args:
+ time_out (int): Timeout in seconds for build operation
+
+ Returns:
+ Self for method chaining
+ """
+ logger.info("Starting container build...")
+
+ super().build(time_out=time_out)
+
+ if self.container:
+ logger.info("Container built successfully, verifying setup...")
+
+ # Verify Python installation
+ exit_code, output = self.container.exec_run(
+ [self.python_path, "--version"]
+ )
+ logger.info(f"Python version check result: {output.decode()}")
+ if exit_code != 0:
+ logger.error(
+ f"Python version check failed with exit code {exit_code}"
+ )
+ raise RuntimeError(
+ f"Python installation verification "
+ f"failed: {output.decode()}"
+ )
+
+ # Install required packages
+ logger.info("Installing required packages...")
+ exit_code, output = self.container.exec_run("apt-get update")
+ if exit_code != 0:
+ logger.error(
+ f"apt-get update failed with "
+ f"exit code {exit_code}: {output.decode()}"
+ )
+ raise RuntimeError(
+ f"Failed to update package lists: {output.decode()}"
+ )
+
+ exit_code, output = self.container.exec_run(
+ "apt-get install -y curl"
+ )
+ if exit_code != 0:
+ logger.error(
+ f"apt-get install curl failed with "
+ f"exit code {exit_code}: {output.decode()}"
+ )
+ raise RuntimeError(
+ f"Failed to install curl: {output.decode()}"
+ )
+
+ # Start API server with explicit Python path
+ logger.info("Starting API server...")
+ exec_result = self.container.exec_run(
+ [self.python_path, "/home/api.py"],
+ detach=True,
+ environment={
+ "PYTHONPATH": str(
+ Path(self.python_path).parent
+ / "lib/python3.10/site-packages"
+ ),
+ "PYTHON_EXECUTABLE": self.python_path,
+ },
+ )
+ logger.info("API server start result: %s", exec_result)
+
+ # Wait for API server to start
+ start_time = time.time()
+ while time.time() - start_time < 10:
+ try:
+ exit_code, curl_result = self.container.exec_run(
+ "curl -s -o /dev/null -w '%{http_code}' http://localhost:8000/docs"
+ )
+ status_code = curl_result.decode().strip()
+ if exit_code == 0 and status_code.startswith('2'):
+ logger.info(
+ f"API server is running "
+ f"(status code: {status_code})"
+ )
+ break
+ else:
+ logger.debug(
+ f"API server not ready yet (status: {status_code})"
+ )
+ except Exception as e:
+ logger.debug("Waiting for API server... %s", e)
+ time.sleep(0.5)
+ else:
+ logger.warning("API server may not be running properly")
+
+ return self
+
+ def exec_python_file(
+ self,
+ local_file_path: str,
+ container_path: Optional[str] = None,
+ args: Optional[List[str]] = None,
+ env: Optional[dict] = None,
+ callback: Optional[Callable[[str], None]] = None,
+ ) -> None:
+ r"""Execute a Python file inside the Docker container.
+
+ Args:
+ local_file_path: Path to the Python file on the local filesystem
+ container_path: Path where the file should be copied in the
+ container If None, the file will be copied to /tmp/
+ args: List of command-line arguments to pass to the Python script
+ env: Additional environment variables to set for the execution
+ callback: Optional function to process each line of output
+ If None, output is printed to stdout
+
+ Raises:
+ RuntimeError: If container is not running
+ FileNotFoundError: If Python file is not found
+ """
+ if not self.container:
+ raise RuntimeError("Container is not running. Call build() first.")
+
+ local_path = Path(local_file_path)
+ if not local_path.exists():
+ raise FileNotFoundError(f"Python file {local_file_path} not found")
+
+ # Determine where to put the file in the container
+ if container_path is None:
+ container_path = f"/tmp/{local_path.name}"
+
+ logger.info(
+ f"Copying {local_file_path} to container at {container_path}"
+ )
+
+ # Copy the file to the container
+ self.container.put_archive(
+ path=str(Path(container_path).parent),
+ data=self._create_archive_from_file(local_path),
+ )
+
+ # Prepare command
+ cmd = [self.python_path, container_path]
+ if args:
+ cmd.extend(args)
+
+ # Prepare environment
+ execution_env = {
+ "PYTHONPATH": "/usr/local/lib/python3.10/site-packages",
+ "PYTHON_EXECUTABLE": self.python_path,
+ }
+ execution_env["PYTHONPATH"] = str(
+ Path(self.python_path).parent / "lib/python3.10/site-packages"
+ )
+ if env:
+ execution_env.update(env)
+
+ logger.info(f"Executing Python file with command: {cmd}")
+
+ # Always use streaming output
+ exec_result = self.container.exec_run(
+ cmd,
+ environment=execution_env,
+ stream=True,
+ demux=True, # Separate stdout and stderr
+ )
+
+ # Handle output streams
+ try:
+ for stdout, stderr in exec_result[1]:
+ if stdout:
+ output = stdout.decode('utf-8')
+ if callback:
+ callback(output)
+ else:
+ print(output, end='')
+
+ if stderr:
+ error = stderr.decode('utf-8')
+ if callback:
+ callback(f"ERROR: {error}")
+ else:
+ print(f"ERROR: {error}", end='', file=sys.stderr)
+ except KeyboardInterrupt:
+ logger.info("Execution interrupted by user")
+ # Could add logic to stop container processes here
+ except Exception as e:
+ logger.error(f"Error during execution: {e}")
+ raise
+
+ def _create_archive_from_file(self, file_path: Union[str, Path]) -> bytes:
+ r"""Create a tar archive from a single file for docker.put_archive().
+
+ Args:
+ file_path: Path to the file to archive
+
+ Returns:
+ bytes: The tar archive as bytes
+ """
+ import io
+ import tarfile
+
+ file_path = Path(file_path)
+ tar_stream = io.BytesIO()
+
+ with tarfile.open(fileobj=tar_stream, mode='w') as tar:
+ tar.add(file_path, arcname=file_path.name)
+
+ tar_stream.seek(0)
+ return tar_stream.read()
diff --git a/examples/runtime/ubuntu_docker_runtime/Dockerfile b/examples/runtime/ubuntu_docker_runtime/Dockerfile
new file mode 100644
index 000000000..0cda80a43
--- /dev/null
+++ b/examples/runtime/ubuntu_docker_runtime/Dockerfile
@@ -0,0 +1,47 @@
+FROM ubuntu:22.04
+
+# Set non-interactive environment
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install basic dependencies
+RUN apt-get update && apt-get install -y \
+ python3 \
+ python3-pip \
+ git \
+ && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /app
+
+# Copy local CAMEL code
+COPY ../../../ ./camel/
+
+# Change to CAMEL directory
+WORKDIR /app/camel
+
+# Install CAMEL and its dependencies
+RUN pip3 install -e ".[all]"
+
+# Copy API service file
+RUN mkdir -p /home
+COPY camel/runtime/api.py /home/api.py
+
+# Set Python path and other environment variables
+ENV PYTHONPATH=/app/camel
+ENV HOST=0.0.0.0
+ENV PORT=8000
+
+# Expose port for API service
+EXPOSE 8000
+
+# Create startup script
+RUN echo '#!/bin/bash\n\
+echo "Starting API service..."\n\
+echo "PYTHONPATH: $PYTHONPATH"\n\
+echo "Current directory: $(pwd)"\n\
+echo "API file exists: $(ls -l /home/api.py)"\n\
+python3 /home/api.py --host 0.0.0.0 --port 8000\n\
+' > /home/start.sh && chmod +x /home/start.sh
+
+# Set default command to start API service with proper host binding
+CMD ["/home/start.sh"]
\ No newline at end of file
diff --git a/examples/runtime/ubuntu_docker_runtime/README.md b/examples/runtime/ubuntu_docker_runtime/README.md
new file mode 100644
index 000000000..0211347af
--- /dev/null
+++ b/examples/runtime/ubuntu_docker_runtime/README.md
@@ -0,0 +1,75 @@
+# Ubuntu Docker Runtime Example
+
+This example demonstrates how to use the CAMEL framework with Docker runtime in an Ubuntu environment, including role-playing capabilities with the Qwen model.
+
+## Prerequisites
+
+- Docker installed and running
+- Python 3.10 or later
+- CAMEL package installed
+- ModelScope API key for Qwen model access
+
+## Directory Structure
+
+```
+ubuntu_docker_runtime/
+โ”œโ”€โ”€ README.md
+โ”œโ”€โ”€ manage_camel_docker.sh
+โ”œโ”€โ”€ Dockerfile
+โ””โ”€โ”€ ubuntu_docker_example.py
+```
+
+## Setup and Running
+
+### 1. Docker Environment Setup
+
+First, build and manage the Docker image:
+
+```bash
+cd camel/examples/runtime/ubuntu_docker_runtime
+chmod +x manage_camel_docker.sh
+./manage_camel_docker.sh build
+```
+
+Verify the image was created:
+```bash
+docker images | grep my-camel
+```
+You should see `my-camel:latest` in the list.
+
+### 2. Test Docker Environment
+
+Run the basic test script to verify the Docker environment:
+```bash
+python test_camel_docker.py
+```
+
+This test will:
+- Start a Docker container with the CAMEL runtime
+- Initialize the API server
+- Run basic operations to verify functionality
+
+### 3. Configure ModelScope API Key
+
+Before running the role-playing test, you need to set up your ModelScope API key in `docker_roleplaying.py`:
+
+```python
+model = ModelFactory.create(
+ model_platform=ModelPlatformType.OPENAI_COMPATIBLE_MODEL,
+ model_type="Qwen/Qwen2.5-72B-Instruct",
+ url='https://api-inference.modelscope.cn/v1/',
+ api_key="YOUR_MODELSCOPE_API_KEY_HERE", # Replace with your API key
+)
+```
+
+### 4. Run Role-Playing Test
+
+After setting up the API key, run the role-playing test:
+```bash
+python ubuntu_docker_example.py
+```
+
+This will:
+- Use the Docker container to run a role-playing scenario
+- Initialize the Qwen model for AI interactions
+- Execute a sample task with AI agents communication
\ No newline at end of file
diff --git a/examples/runtime/ubuntu_docker_runtime/manage_camel_docker.sh b/examples/runtime/ubuntu_docker_runtime/manage_camel_docker.sh
new file mode 100755
index 000000000..af8cf271c
--- /dev/null
+++ b/examples/runtime/ubuntu_docker_runtime/manage_camel_docker.sh
@@ -0,0 +1,151 @@
+#!/bin/bash
+
+# Set image name and tag
+IMAGE_NAME="my-camel"
+TAG="latest"
+FULL_NAME="${IMAGE_NAME}:${TAG}"
+
+# Get CAMEL project root directory absolute path
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+CAMEL_ROOT="$( cd "$SCRIPT_DIR/../../.." && pwd )"
+
+# Display help information
+show_help() {
+ echo "CAMEL Docker Image Management Script"
+ echo "Usage: $0 [options]"
+ echo "Options:"
+ echo " build - Build new Docker image"
+ echo " clean - Clean related containers and images"
+ echo " rebuild - Clean and rebuild image"
+ echo " list - List related containers and images"
+ echo " help - Show this help message"
+}
+
+# Build Docker image
+build_image() {
+ echo "Starting Docker image build..."
+ echo "Using CAMEL source path: $CAMEL_ROOT"
+
+ # Build in temporary directory
+ TEMP_DIR=$(mktemp -d)
+ echo "Creating temporary build directory: $TEMP_DIR"
+
+ # Copy necessary files to temporary directory
+ cp "$SCRIPT_DIR/Dockerfile" "$TEMP_DIR/"
+ cp -r "$CAMEL_ROOT" "$TEMP_DIR/camel_source"
+
+ # Ensure API file exists
+ if [ ! -f "$CAMEL_ROOT/camel/runtime/api.py" ]; then
+ echo "Error: API file not found at $CAMEL_ROOT/camel/runtime/api.py"
+ exit 1
+ fi
+
+ # Copy API file to a known location
+ mkdir -p "$TEMP_DIR/api"
+ cp "$CAMEL_ROOT/camel/runtime/api.py" "$TEMP_DIR/api/"
+
+ # Modify Dockerfile COPY commands - fix the sed command
+ sed -i '' 's|COPY ../../../|COPY camel_source/|g' "$TEMP_DIR/Dockerfile"
+ sed -i '' 's|COPY camel/runtime/api.py|COPY api/api.py|g' "$TEMP_DIR/Dockerfile"
+
+ # Build in temporary directory
+ (cd "$TEMP_DIR" && docker build -t ${FULL_NAME} .)
+
+ BUILD_RESULT=$?
+
+ # Clean temporary directory
+ rm -rf "$TEMP_DIR"
+
+ if [ $BUILD_RESULT -eq 0 ]; then
+ echo "Docker image build successful!"
+ echo "Image name: ${FULL_NAME}"
+ echo ""
+ echo "You can use this image in your code as follows:"
+ echo "runtime = DockerRuntime(\"${FULL_NAME}\")"
+ else
+ echo "Docker image build failed!"
+ exit 1
+ fi
+}
+
+# Add this function after build_image
+check_container() {
+ container_id=$1
+ echo "Checking container logs..."
+ docker logs $container_id
+
+ echo "Checking container status..."
+ docker inspect $container_id --format='{{.State.Status}}'
+
+ echo "Checking if API is responding..."
+ curl -v http://localhost:8000/docs
+}
+
+# Clean containers and images
+clean() {
+ echo "Starting cleanup..."
+
+ # Stop and remove related containers
+ echo "Finding and stopping related containers..."
+ containers=$(docker ps -a --filter "ancestor=${FULL_NAME}" --format "{{.ID}}")
+ if [ ! -z "$containers" ]; then
+ echo "Found following containers:"
+ docker ps -a --filter "ancestor=${FULL_NAME}"
+ echo "Stopping and removing containers..."
+ docker stop $containers 2>/dev/null
+ docker rm $containers 2>/dev/null
+ else
+ echo "No related containers found"
+ fi
+
+ # Remove image
+ echo "Removing image..."
+ if docker images ${FULL_NAME} --format "{{.ID}}" | grep -q .; then
+ docker rmi ${FULL_NAME}
+ echo "Image removed"
+ else
+ echo "No related image found"
+ fi
+
+ # Clean unused images and build cache
+ echo "Cleaning unused images and build cache..."
+ docker system prune -f
+
+ echo "Cleanup complete"
+}
+
+# List related resources
+list_resources() {
+ echo "Related Docker resources:"
+ echo ""
+ echo "Container list:"
+ docker ps -a --filter "ancestor=${FULL_NAME}"
+ echo ""
+ echo "Image information:"
+ docker images ${FULL_NAME}
+}
+
+# Main program
+case "$1" in
+ "build")
+ build_image
+ ;;
+ "clean")
+ clean
+ ;;
+ "rebuild")
+ clean
+ build_image
+ ;;
+ "list")
+ list_resources
+ ;;
+ "help"|"")
+ show_help
+ ;;
+ *)
+ echo "Unknown option: $1"
+ show_help
+ exit 1
+ ;;
+esac
\ No newline at end of file
diff --git a/examples/runtime/ubuntu_docker_runtime/ubuntu_docker_example.py b/examples/runtime/ubuntu_docker_runtime/ubuntu_docker_example.py
new file mode 100644
index 000000000..2711b1796
--- /dev/null
+++ b/examples/runtime/ubuntu_docker_runtime/ubuntu_docker_example.py
@@ -0,0 +1,51 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import sys
+
+from camel.runtime import UbuntuDockerRuntime
+
+
+def main():
+ r"""Main function to demonstrate Ubuntu Docker Runtime."""
+ try:
+ # Get OpenAI API key from environment
+ openai_api_key = os.environ.get("OPENAI_API_KEY")
+
+ # Initialize the runtime
+ runtime = UbuntuDockerRuntime(
+ image="my-camel:latest",
+ python_path="/usr/bin/python3",
+ ports=0,
+ environment={
+ "HOST": "0.0.0.0",
+ "OPENAI_API_KEY": openai_api_key, # Pass API key to container
+ },
+ )
+
+ runtime.build()
+ script_dir = os.path.dirname(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ )
+ script_path = script_dir + "/ai_society/role_playing.py"
+ runtime.exec_python_file(local_file_path=script_path)
+
+ except Exception as e:
+ print(f"Error: {e}")
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()