๐ซ CAMEL: The first and the best multi-agent framework. Finding the Scaling Law of Agents. https://www.camel-ai.org
Python agentai-societiesartificial-intelligencecommunicative-aicooperative-aideep-learninglarge-language-modelsmulti-agent-systemsnatural-language-processing
feat: Add Ubuntu docker runtime (#1952)
jjyaoao4f83f7cb1931
diff --git a/camel/runtime/__init__.py b/camel/runtime/__init__.pyindex 024b7b066..3159a1c9d 100644--- a/camel/runtime/__init__.py+++ b/camel/runtime/__init__.py@@ -16,6 +16,7 @@ from .configs import TaskConfigfrom .docker_runtime import DockerRuntimefrom .llm_guard_runtime import LLMGuardRuntimefrom .remote_http_runtime import RemoteHttpRuntime+from .ubuntu_docker_runtime import UbuntuDockerRuntime# TODO: Add Celery Runtime to support distributed computing,# Rate Limiting, Load Balancing, etc.@@ -26,4 +27,5 @@ __all__ = ["RemoteHttpRuntime","LLMGuardRuntime","TaskConfig",+ "UbuntuDockerRuntime",]diff --git a/camel/runtime/ubuntu_docker_runtime.py b/camel/runtime/ubuntu_docker_runtime.pynew file mode 100644index 000000000..f2149d583--- /dev/null+++ b/camel/runtime/ubuntu_docker_runtime.py@@ -0,0 +1,340 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import logging+import sys+import time+from pathlib import Path+from typing import Callable, List, Optional, Union++from camel.runtime.docker_runtime import DockerRuntime+from camel.toolkits import FunctionTool++logger = logging.getLogger(__name__)+++class UbuntuDockerRuntime(DockerRuntime):+ r"""A specialized Docker runtime for Ubuntu-based environments.++ This runtime includes specific configurations and setup for Ubuntu+ containers, including proper Python path handling and environment setup.+ It provides methods for executing Python files, managing the container+ lifecycle, and handling file operations within the Ubuntu container.++ Attributes:+ python_path (str): Path to the Python interpreter in the container+ docker_config (dict): Configuration dict for Docker container setup+ """++ def __init__(+ self,+ image: str,+ port: int = 0,+ remove: bool = True,+ python_path: str = "/usr/bin/python3",+ **kwargs,+ ):+ r"""Initialize the Ubuntu Docker Runtime.++ Args:+ image (str): Docker image name to use+ port (int, optional): Port to expose. Defaults to 0 (random port)+ remove (bool, optional): Whether to remove container after use.+ Defaults to True+ python_path (str, optional): Path to Python interpreter.+ Defaults to "/usr/bin/python3"+ **kwargs: Additional arguments passed to DockerRuntime+ """+ super().__init__(image=image, port=port, remove=remove, **kwargs)++ self.python_path = python_path+ logger.info(+ f"Initializing UbuntuDockerRuntime with python_path: {python_path}"+ )++ # Set default environment variables for Ubuntu+ self.docker_config.setdefault("environment", {})+ self.docker_config["environment"].update(+ {+ "PYTHON_PATH": python_path,+ "PYTHON_EXECUTABLE": python_path,+ "PATH": "/usr/local/bin:/usr/bin:/bin",+ "PYTHONUNBUFFERED": "1",+ }+ )+ logger.info(+ f"Environment variables set: {self.docker_config['environment']}"+ )++ # Add default working directory+ self.docker_config.setdefault("working_dir", "/app")++ # Setup default volume mounts+ self._setup_default_mounts()++ def add(+ self,+ funcs: Union[FunctionTool, List[FunctionTool]],+ entrypoint: str,+ redirect_stdout: bool = False,+ arguments: Optional[dict] = None,+ ) -> "UbuntuDockerRuntime":+ r"""Add functions to the runtime with Ubuntu-specific modifications.++ Args:+ funcs: Function(s) to add to the runtime+ entrypoint: Entry point for function execution+ redirect_stdout: Whether to redirect stdout+ arguments: Optional arguments for function execution++ Returns:+ Self for method chaining+ """+ if not isinstance(funcs, list):+ funcs = [funcs]++ # Modify the code execution command to use python3+ for func in funcs:+ logger.info(f"Processing function: {func.get_function_name()}")+ if hasattr(func, 'command'):+ logger.info(f"Original command: {func.command}")+ if isinstance(func.command, list):+ if 'python' in func.command:+ idx = func.command.index('python')+ func.command[idx] = self.python_path+ logger.info(f"Modified command: {func.command}")+ else:+ logger.info(+ f"No command attribute found for function "+ f"{func.get_function_name()}"+ )++ super().add(funcs, entrypoint, redirect_stdout, arguments)+ return self++ def _setup_default_mounts(self):+ r"""Setup default volume mounts for the container.++ This method can be extended to add Ubuntu-specific volume mounts.+ """+ pass++ def build(self, time_out: int = 15) -> "UbuntuDockerRuntime":+ r"""Build and initialize the Ubuntu container with proper setup.++ Args:+ time_out (int): Timeout in seconds for build operation++ Returns:+ Self for method chaining+ """+ logger.info("Starting container build...")++ super().build(time_out=time_out)++ if self.container:+ logger.info("Container built successfully, verifying setup...")++ # Verify Python installation+ exit_code, output = self.container.exec_run(+ [self.python_path, "--version"]+ )+ logger.info(f"Python version check result: {output.decode()}")+ if exit_code != 0:+ logger.error(+ f"Python version check failed with exit code {exit_code}"+ )+ raise RuntimeError(+ f"Python installation verification "+ f"failed: {output.decode()}"+ )++ # Install required packages+ logger.info("Installing required packages...")+ exit_code, output = self.container.exec_run("apt-get update")+ if exit_code != 0:+ logger.error(+ f"apt-get update failed with "+ f"exit code {exit_code}: {output.decode()}"+ )+ raise RuntimeError(+ f"Failed to update package lists: {output.decode()}"+ )++ exit_code, output = self.container.exec_run(+ "apt-get install -y curl"+ )+ if exit_code != 0:+ logger.error(+ f"apt-get install curl failed with "+ f"exit code {exit_code}: {output.decode()}"+ )+ raise RuntimeError(+ f"Failed to install curl: {output.decode()}"+ )++ # Start API server with explicit Python path+ logger.info("Starting API server...")+ exec_result = self.container.exec_run(+ [self.python_path, "/home/api.py"],+ detach=True,+ environment={+ "PYTHONPATH": str(+ Path(self.python_path).parent+ / "lib/python3.10/site-packages"+ ),+ "PYTHON_EXECUTABLE": self.python_path,+ },+ )+ logger.info("API server start result: %s", exec_result)++ # Wait for API server to start+ start_time = time.time()+ while time.time() - start_time < 10:+ try:+ exit_code, curl_result = self.container.exec_run(+ "curl -s -o /dev/null -w '%{http_code}' http://localhost:8000/docs"+ )+ status_code = curl_result.decode().strip()+ if exit_code == 0 and status_code.startswith('2'):+ logger.info(+ f"API server is running "+ f"(status code: {status_code})"+ )+ break+ else:+ logger.debug(+ f"API server not ready yet (status: {status_code})"+ )+ except Exception as e:+ logger.debug("Waiting for API server... %s", e)+ time.sleep(0.5)+ else:+ logger.warning("API server may not be running properly")++ return self++ def exec_python_file(+ self,+ local_file_path: str,+ container_path: Optional[str] = None,+ args: Optional[List[str]] = None,+ env: Optional[dict] = None,+ callback: Optional[Callable[[str], None]] = None,+ ) -> None:+ r"""Execute a Python file inside the Docker container.++ Args:+ local_file_path: Path to the Python file on the local filesystem+ container_path: Path where the file should be copied in the+ container If None, the file will be copied to /tmp/+ args: List of command-line arguments to pass to the Python script+ env: Additional environment variables to set for the execution+ callback: Optional function to process each line of output+ If None, output is printed to stdout++ Raises:+ RuntimeError: If container is not running+ FileNotFoundError: If Python file is not found+ """+ if not self.container:+ raise RuntimeError("Container is not running. Call build() first.")++ local_path = Path(local_file_path)+ if not local_path.exists():+ raise FileNotFoundError(f"Python file {local_file_path} not found")++ # Determine where to put the file in the container+ if container_path is None:+ container_path = f"/tmp/{local_path.name}"++ logger.info(+ f"Copying {local_file_path} to container at {container_path}"+ )++ # Copy the file to the container+ self.container.put_archive(+ path=str(Path(container_path).parent),+ data=self._create_archive_from_file(local_path),+ )++ # Prepare command+ cmd = [self.python_path, container_path]+ if args:+ cmd.extend(args)++ # Prepare environment+ execution_env = {+ "PYTHONPATH": "/usr/local/lib/python3.10/site-packages",+ "PYTHON_EXECUTABLE": self.python_path,+ }+ execution_env["PYTHONPATH"] = str(+ Path(self.python_path).parent / "lib/python3.10/site-packages"+ )+ if env:+ execution_env.update(env)++ logger.info(f"Executing Python file with command: {cmd}")++ # Always use streaming output+ exec_result = self.container.exec_run(+ cmd,+ environment=execution_env,+ stream=True,+ demux=True, # Separate stdout and stderr+ )++ # Handle output streams+ try:+ for stdout, stderr in exec_result[1]:+ if stdout:+ output = stdout.decode('utf-8')+ if callback:+ callback(output)+ else:+ print(output, end='')++ if stderr:+ error = stderr.decode('utf-8')+ if callback:+ callback(f"ERROR: {error}")+ else:+ print(f"ERROR: {error}", end='', file=sys.stderr)+ except KeyboardInterrupt:+ logger.info("Execution interrupted by user")+ # Could add logic to stop container processes here+ except Exception as e:+ logger.error(f"Error during execution: {e}")+ raise++ def _create_archive_from_file(self, file_path: Union[str, Path]) -> bytes:+ r"""Create a tar archive from a single file for docker.put_archive().++ Args:+ file_path: Path to the file to archive++ Returns:+ bytes: The tar archive as bytes+ """+ import io+ import tarfile++ file_path = Path(file_path)+ tar_stream = io.BytesIO()++ with tarfile.open(fileobj=tar_stream, mode='w') as tar:+ tar.add(file_path, arcname=file_path.name)++ tar_stream.seek(0)+ return tar_stream.read()diff --git a/examples/runtime/ubuntu_docker_runtime/Dockerfile b/examples/runtime/ubuntu_docker_runtime/Dockerfilenew file mode 100644index 000000000..0cda80a43--- /dev/null+++ b/examples/runtime/ubuntu_docker_runtime/Dockerfile@@ -0,0 +1,47 @@+FROM ubuntu:22.04++# Set non-interactive environment+ENV DEBIAN_FRONTEND=noninteractive++# Install basic dependencies+RUN apt-get update && apt-get install -y \+ python3 \+ python3-pip \+ git \+ && rm -rf /var/lib/apt/lists/*++# Set working directory+WORKDIR /app++# Copy local CAMEL code+COPY ../../../ ./camel/++# Change to CAMEL directory+WORKDIR /app/camel++# Install CAMEL and its dependencies+RUN pip3 install -e ".[all]"++# Copy API service file+RUN mkdir -p /home+COPY camel/runtime/api.py /home/api.py++# Set Python path and other environment variables+ENV PYTHONPATH=/app/camel+ENV HOST=0.0.0.0+ENV PORT=8000++# Expose port for API service+EXPOSE 8000++# Create startup script+RUN echo '#!/bin/bash\n\+echo "Starting API service..."\n\+echo "PYTHONPATH: $PYTHONPATH"\n\+echo "Current directory: $(pwd)"\n\+echo "API file exists: $(ls -l /home/api.py)"\n\+python3 /home/api.py --host 0.0.0.0 --port 8000\n\+' > /home/start.sh && chmod +x /home/start.sh++# Set default command to start API service with proper host binding+CMD ["/home/start.sh"]\ No newline at end of filediff --git a/examples/runtime/ubuntu_docker_runtime/README.md b/examples/runtime/ubuntu_docker_runtime/README.mdnew file mode 100644index 000000000..0211347af--- /dev/null+++ b/examples/runtime/ubuntu_docker_runtime/README.md@@ -0,0 +1,75 @@+# Ubuntu Docker Runtime Example++This example demonstrates how to use the CAMEL framework with Docker runtime in an Ubuntu environment, including role-playing capabilities with the Qwen model.++## Prerequisites++- Docker installed and running+- Python 3.10 or later+- CAMEL package installed+- ModelScope API key for Qwen model access++## Directory Structure++```+ubuntu_docker_runtime/+โโโ README.md+โโโ manage_camel_docker.sh+โโโ Dockerfile+โโโ ubuntu_docker_example.py+```++## Setup and Running++### 1. Docker Environment Setup++First, build and manage the Docker image:++```bash+cd camel/examples/runtime/ubuntu_docker_runtime+chmod +x manage_camel_docker.sh+./manage_camel_docker.sh build+```++Verify the image was created:+```bash+docker images | grep my-camel+```+You should see `my-camel:latest` in the list.++### 2. Test Docker Environment++Run the basic test script to verify the Docker environment:+```bash+python test_camel_docker.py+```++This test will:+- Start a Docker container with the CAMEL runtime+- Initialize the API server+- Run basic operations to verify functionality++### 3. Configure ModelScope API Key++Before running the role-playing test, you need to set up your ModelScope API key in `docker_roleplaying.py`:++```python+model = ModelFactory.create(+ model_platform=ModelPlatformType.OPENAI_COMPATIBLE_MODEL,+ model_type="Qwen/Qwen2.5-72B-Instruct",+ url='https://api-inference.modelscope.cn/v1/',+ api_key="YOUR_MODELSCOPE_API_KEY_HERE", # Replace with your API key+)+```++### 4. Run Role-Playing Test++After setting up the API key, run the role-playing test:+```bash+python ubuntu_docker_example.py+```++This will:+- Use the Docker container to run a role-playing scenario+- Initialize the Qwen model for AI interactions+- Execute a sample task with AI agents communication\ No newline at end of filediff --git a/examples/runtime/ubuntu_docker_runtime/manage_camel_docker.sh b/examples/runtime/ubuntu_docker_runtime/manage_camel_docker.shnew file mode 100755index 000000000..af8cf271c--- /dev/null+++ b/examples/runtime/ubuntu_docker_runtime/manage_camel_docker.sh@@ -0,0 +1,151 @@+#!/bin/bash++# Set image name and tag+IMAGE_NAME="my-camel"+TAG="latest"+FULL_NAME="${IMAGE_NAME}:${TAG}"++# Get CAMEL project root directory absolute path+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"+CAMEL_ROOT="$( cd "$SCRIPT_DIR/../../.." && pwd )"++# Display help information+show_help() {+ echo "CAMEL Docker Image Management Script"+ echo "Usage: $0 [options]"+ echo "Options:"+ echo " build - Build new Docker image"+ echo " clean - Clean related containers and images"+ echo " rebuild - Clean and rebuild image"+ echo " list - List related containers and images"+ echo " help - Show this help message"+}++# Build Docker image+build_image() {+ echo "Starting Docker image build..."+ echo "Using CAMEL source path: $CAMEL_ROOT"++ # Build in temporary directory+ TEMP_DIR=$(mktemp -d)+ echo "Creating temporary build directory: $TEMP_DIR"++ # Copy necessary files to temporary directory+ cp "$SCRIPT_DIR/Dockerfile" "$TEMP_DIR/"+ cp -r "$CAMEL_ROOT" "$TEMP_DIR/camel_source"++ # Ensure API file exists+ if [ ! -f "$CAMEL_ROOT/camel/runtime/api.py" ]; then+ echo "Error: API file not found at $CAMEL_ROOT/camel/runtime/api.py"+ exit 1+ fi++ # Copy API file to a known location+ mkdir -p "$TEMP_DIR/api"+ cp "$CAMEL_ROOT/camel/runtime/api.py" "$TEMP_DIR/api/"++ # Modify Dockerfile COPY commands - fix the sed command+ sed -i '' 's|COPY ../../../|COPY camel_source/|g' "$TEMP_DIR/Dockerfile"+ sed -i '' 's|COPY camel/runtime/api.py|COPY api/api.py|g' "$TEMP_DIR/Dockerfile"++ # Build in temporary directory+ (cd "$TEMP_DIR" && docker build -t ${FULL_NAME} .)++ BUILD_RESULT=$?++ # Clean temporary directory+ rm -rf "$TEMP_DIR"++ if [ $BUILD_RESULT -eq 0 ]; then+ echo "Docker image build successful!"+ echo "Image name: ${FULL_NAME}"+ echo ""+ echo "You can use this image in your code as follows:"+ echo "runtime = DockerRuntime(\"${FULL_NAME}\")"+ else+ echo "Docker image build failed!"+ exit 1+ fi+}++# Add this function after build_image+check_container() {+ container_id=$1+ echo "Checking container logs..."+ docker logs $container_id++ echo "Checking container status..."+ docker inspect $container_id --format='{{.State.Status}}'++ echo "Checking if API is responding..."+ curl -v http://localhost:8000/docs+}++# Clean containers and images+clean() {+ echo "Starting cleanup..."++ # Stop and remove related containers+ echo "Finding and stopping related containers..."+ containers=$(docker ps -a --filter "ancestor=${FULL_NAME}" --format "{{.ID}}")+ if [ ! -z "$containers" ]; then+ echo "Found following containers:"+ docker ps -a --filter "ancestor=${FULL_NAME}"+ echo "Stopping and removing containers..."+ docker stop $containers 2>/dev/null+ docker rm $containers 2>/dev/null+ else+ echo "No related containers found"+ fi++ # Remove image+ echo "Removing image..."+ if docker images ${FULL_NAME} --format "{{.ID}}" | grep -q .; then+ docker rmi ${FULL_NAME}+ echo "Image removed"+ else+ echo "No related image found"+ fi++ # Clean unused images and build cache+ echo "Cleaning unused images and build cache..."+ docker system prune -f++ echo "Cleanup complete"+}++# List related resources+list_resources() {+ echo "Related Docker resources:"+ echo ""+ echo "Container list:"+ docker ps -a --filter "ancestor=${FULL_NAME}"+ echo ""+ echo "Image information:"+ docker images ${FULL_NAME}+}++# Main program+case "$1" in+ "build")+ build_image+ ;;+ "clean")+ clean+ ;;+ "rebuild")+ clean+ build_image+ ;;+ "list")+ list_resources+ ;;+ "help"|"")+ show_help+ ;;+ *)+ echo "Unknown option: $1"+ show_help+ exit 1+ ;;+esac\ No newline at end of filediff --git a/examples/runtime/ubuntu_docker_runtime/ubuntu_docker_example.py b/examples/runtime/ubuntu_docker_runtime/ubuntu_docker_example.pynew file mode 100644index 000000000..2711b1796--- /dev/null+++ b/examples/runtime/ubuntu_docker_runtime/ubuntu_docker_example.py@@ -0,0 +1,51 @@+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========++import os+import sys++from camel.runtime import UbuntuDockerRuntime+++def main():+ r"""Main function to demonstrate Ubuntu Docker Runtime."""+ try:+ # Get OpenAI API key from environment+ openai_api_key = os.environ.get("OPENAI_API_KEY")++ # Initialize the runtime+ runtime = UbuntuDockerRuntime(+ image="my-camel:latest",+ python_path="/usr/bin/python3",+ ports=0,+ environment={+ "HOST": "0.0.0.0",+ "OPENAI_API_KEY": openai_api_key, # Pass API key to container+ },+ )++ runtime.build()+ script_dir = os.path.dirname(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ )+ script_path = script_dir + "/ai_society/role_playing.py"+ runtime.exec_python_file(local_file_path=script_path)++ except Exception as e:+ print(f"Error: {e}")+ sys.exit(1)+++if __name__ == "__main__":+ main()