Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

27.3 KB
Raw Blame
#
# Copyright 2025 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Filesystem service adapter implementation.

Implementation of FilesystemService that adapts openapi-python-client generated FilesystemApi.
This adapter handles file operations within sandboxes using the auto-generated API client.
"""

import json
import logging
import os
import time
from collections.abc import AsyncGenerator, AsyncIterator
from io import IOBase, TextIOBase
from typing import TypedDict

import httpx

from opensandbox._httpx import build_async_redirect_client_options
from opensandbox.adapters.converter.exception_converter import (
    ExceptionConverter,
)
from opensandbox.adapters.converter.filesystem_model_converter import (
    FilesystemModelConverter,
)
from opensandbox.adapters.converter.response_handler import (
    extract_request_id,
    handle_api_error,
)
from opensandbox.adapters.download_response import (
    create_read_bytes_response,
    iter_async_response_bytes,
)
from opensandbox.adapters.filesystem_identity import filesystem_identity_path
from opensandbox.config import ConnectionConfig
from opensandbox.exceptions import InvalidArgumentException, SandboxApiException
from opensandbox.models.filesystem import (
    AsyncReadBytesStream,
    ContentReplaceEntry,
    ContentReplaceResult,
    DirectoryListEntry,
    EntryInfo,
    MoveEntry,
    ReadBytesResponse,
    SearchEntry,
    SetPermissionEntry,
    WriteEntry,
)
from opensandbox.models.sandboxes import SandboxEndpoint
from opensandbox.services.filesystem import Filesystem

logger = logging.getLogger(__name__)


def _multipart_header_filename(filename: str) -> str:
    return (
        filename.replace("\\", "\\\\")
        .replace('"', r'\"')
        .replace("\r", "_")
        .replace("\n", "_")
    )


def _rewind_seekable_stream(stream: IOBase) -> None:
    if not stream.seekable():
        return
    stream.seek(0)


class _DownloadRequest(TypedDict):
    url: str
    params: dict[str, str]
    headers: dict[str, str]


class FilesystemAdapter(Filesystem):
    """
    Implementation of FilesystemService that provides comprehensive file system operations.

    This adapter handles file operations within sandboxes using optimized approaches
    for different operation types - API calls for standard operations and direct HTTP
    for file upload/download operations requiring special handling.

    All HTTP clients created by this adapter share `ConnectionConfig.transport`.
    """

    FILESYSTEM_UPLOAD_PATH = "/files/upload"
    FILESYSTEM_DOWNLOAD_PATH = "/files/download"

    def __init__(
        self, connection_config: ConnectionConfig, execd_endpoint: SandboxEndpoint,
        *, _identity_prefix: str = "",
    ) -> None:
        """
        Initialize the filesystem service adapter.

        Args:
            connection_config: Connection configuration (shared transport, headers, timeouts)
            execd_endpoint: Execd endpoint information for direct HTTP calls
        """
        self.connection_config = connection_config
        self.execd_endpoint = execd_endpoint
        self._identity_prefix = _identity_prefix
        from opensandbox.api.execd import Client

        base_url = self._get_execd_base_url()
        timeout_seconds = self.connection_config.request_timeout.total_seconds()
        timeout = httpx.Timeout(timeout_seconds)
        headers = self.execd_endpoint.build_request_headers(self.connection_config)

        self._httpx_client = httpx.AsyncClient(
            base_url=base_url,
            headers=headers,
            timeout=timeout,
            transport=self.connection_config.transport,
            **build_async_redirect_client_options(self.connection_config, base_url),
        )

        self._client = Client(
            base_url=base_url,
            timeout=timeout,
            follow_redirects=self.connection_config.follow_redirects,
        )
        self._client.set_async_httpx_client(self._httpx_client)

    def with_identity(self, uid: int, gid: int) -> "FilesystemAdapter":
        """Create an independent filesystem client with explicit Linux credentials."""
        return FilesystemAdapter(
            self.connection_config,
            self.execd_endpoint,
            _identity_prefix=filesystem_identity_path(uid, gid),
        )

    def _get_execd_base_url(self) -> str:
        protocol = self.connection_config.protocol
        return f"{protocol}://{self.execd_endpoint.endpoint.rstrip('/')}{self._identity_prefix}"

    async def _get_httpx_client(self) -> httpx.AsyncClient:
        """Return adapter-owned httpx client for execd."""
        return self._httpx_client

    async def _get_client(self):
        """Return the client for execd API."""
        return self._client

    def _get_execd_url(self, path: str) -> str:
        """Build URL for execd endpoint."""
        return f"{self._get_execd_base_url()}{path}"

    async def read_file(
        self,
        path: str,
        *,
        encoding: str = "utf-8",
        range_header: str | None = None,
        offset: int | None = None,
        limit: int | None = None,
    ) -> str:
        """Read file content as string via HTTP API."""
        content = await self.read_bytes(path, range_header=range_header, offset=offset, limit=limit)
        return content.decode(encoding)

    async def read_bytes(
        self,
        path: str,
        *,
        range_header: str | None = None,
        offset: int | None = None,
        limit: int | None = None,
    ) -> bytes:
        """Read file content as bytes."""
        return (
            await self.read_bytes_detailed(
                path, range_header=range_header, offset=offset, limit=limit
            )
        ).body

    async def read_bytes_detailed(
        self,
        path: str,
        *,
        range_header: str | None = None,
        offset: int | None = None,
        limit: int | None = None,
    ) -> ReadBytesResponse[bytes]:
        """Read file bytes and HTTP response metadata.

        Args:
            path: Path to the file to read
            range_header: Optional range header for partial content requests.
                Mutually exclusive with offset/limit.
            offset: Starting line number (1-based) for line-based reading.
                Mutually exclusive with range_header.
            limit: Number of lines to return for line-based reading.
                Mutually exclusive with range_header.

        Returns:
            File bytes together with HTTP status and range metadata

        Raises:
            SandboxApiException: If the read operation fails
        """
        logger.debug(f"Reading file as bytes: {path}")
        try:
            request_data = self._build_download_request(
                path, range_header, offset=offset, limit=limit
            )
            client = await self._get_httpx_client()

            response = await client.get(
                request_data["url"],
                headers=request_data["headers"],
                params=request_data["params"],
            )
            response.raise_for_status()
            return create_read_bytes_response(
                response,
                response.content,
                line_based=offset is not None or limit is not None,
            )
        except Exception as e:
            logger.error(f"Failed to read file {path}", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def read_bytes_stream(
        self,
        path: str,
        *,
        chunk_size: int = 64 * 1024,
        range_header: str | None = None,
        offset: int | None = None,
        limit: int | None = None,
    ) -> AsyncGenerator[bytes, None]:
        """Stream file content as byte chunks."""
        response = await self.read_bytes_stream_detailed(
            path,
            chunk_size=chunk_size,
            range_header=range_header,
            offset=offset,
            limit=limit,
        )

        async def _iter() -> AsyncGenerator[bytes, None]:
            try:
                yield b""
                async for chunk in response.body:
                    yield chunk
            finally:
                await response.body.aclose()

        # Start the generator so its finally block also runs when it is
        # closed or dropped before the caller reads the first chunk.
        chunks = _iter()
        await anext(chunks)
        return chunks

    async def read_bytes_stream_detailed(
        self,
        path: str,
        *,
        chunk_size: int = 64 * 1024,
        range_header: str | None = None,
        offset: int | None = None,
        limit: int | None = None,
    ) -> ReadBytesResponse[AsyncReadBytesStream]:
        """Stream file content as bytes chunks via HTTP (true streaming)."""
        logger.debug(f"Streaming file as bytes: {path} (chunk_size={chunk_size})")
        try:
            request_data = self._build_download_request(
                path, range_header, offset=offset, limit=limit
            )
            client = await self._get_httpx_client()

            url = request_data["url"]
            params = request_data["params"]
            headers = request_data["headers"]

            request = client.build_request(
                "GET",
                url,
                headers=headers,
                params=params,
            )

            response = await client.send(request, stream=True)

            if response.status_code >= 300:
                try:
                    await response.aread()
                finally:
                    await response.aclose()

                raise SandboxApiException(
                    f"Failed to stream file {path}: {response.status_code}",
                    status_code=response.status_code,
                    request_id=extract_request_id(response.headers),
                )
            return create_read_bytes_response(
                response,
                iter_async_response_bytes(response, chunk_size),
                line_based=offset is not None or limit is not None,
            )
        except Exception as e:
            logger.error(f"Failed to stream file {path}", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def write_files(self, entries: list[WriteEntry]) -> None:
        """Write multiple files in a single operation using multipart upload.

        Uses chunked transfer encoding for direct execd access to bypass
        ingress proxy body size limits. Uses Content-Length when going
        through the server proxy, which does not support chunked multipart.
        """
        if not entries:
            return

        logger.debug(f"Writing {len(entries)} files")

        try:
            client = await self._get_httpx_client()
            url = self._get_execd_url(self.FILESYSTEM_UPLOAD_PATH)

            if self.connection_config.use_server_proxy:
                response = await self._write_files_with_content_length(
                    url, client, entries
                )
            else:
                response = await self._write_files_chunked(url, client, entries)

            response.raise_for_status()
        except Exception as e:
            logger.error(f"Failed to write {len(entries)} files", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def _write_files_with_content_length(
        self,
        url: str,
        client: httpx.AsyncClient,
        entries: list[WriteEntry],
    ) -> httpx.Response:
        """Upload via httpx ``files=`` parameter with Content-Length.

        Used when going through the server proxy, which requires Content-Length
        for multipart requests.
        """
        multipart_parts = []
        for entry in entries:
            if not entry.path:
                raise InvalidArgumentException("File path cannot be null")
            if entry.data is None:
                raise InvalidArgumentException("File data cannot be null")

            metadata = {
                "path": entry.path,
                "owner": entry.owner,
                "group": entry.group,
                "mode": entry.mode,
            }
            metadata_json = json.dumps(metadata)
            multipart_parts.append(
                ("metadata", ("metadata", metadata_json, "application/json"))
            )

            content: bytes | str | IOBase
            content_type: str

            if isinstance(entry.data, bytes):
                content = entry.data
                content_type = "application/octet-stream"
            elif isinstance(entry.data, str):
                encoding = entry.encoding or "utf-8"
                content = entry.data.encode(encoding)
                content_type = f"text/plain; charset={encoding}"
            elif isinstance(entry.data, IOBase):
                if isinstance(entry.data, TextIOBase):
                    raise InvalidArgumentException(
                        "File stream must be binary (opened with 'rb'). Text streams are not supported."
                    )
                content = entry.data
                content_type = "application/octet-stream"
            else:
                raise InvalidArgumentException(
                    f"Unsupported file data type: {type(entry.data)}"
                )

            multipart_parts.append(("file", (entry.path, content, content_type)))

        return await client.post(
            url,
            files=multipart_parts,
            follow_redirects=False,
        )

    async def _write_files_chunked(
        self,
        url: str,
        client: httpx.AsyncClient,
        entries: list[WriteEntry],
    ) -> httpx.Response:
        """Upload via chunked transfer encoding.

        Used for direct execd access to bypass ingress proxy body size limits.
        """
        boundary = f"opensandbox_{os.urandom(8).hex()}_{int(time.time())}"

        async def _body() -> AsyncIterator[bytes]:
            for entry in entries:
                if not entry.path:
                    raise InvalidArgumentException("File path cannot be null")
                if entry.data is None:
                    raise InvalidArgumentException("File data cannot be null")

                metadata = {
                    "path": entry.path,
                    "owner": entry.owner,
                    "group": entry.group,
                    "mode": entry.mode,
                }
                metadata_json = json.dumps(metadata)

                content: bytes | str | IOBase
                content_type: str

                if isinstance(entry.data, bytes):
                    content = entry.data
                    content_type = "application/octet-stream"
                elif isinstance(entry.data, str):
                    encoding = entry.encoding or "utf-8"
                    content = entry.data.encode(encoding)
                    content_type = f"text/plain; charset={encoding}"
                elif isinstance(entry.data, IOBase):
                    if isinstance(entry.data, TextIOBase):
                        raise InvalidArgumentException(
                            "File stream must be binary (opened with 'rb'). Text streams are not supported."
                        )
                    content = entry.data
                    _rewind_seekable_stream(content)
                    content_type = "application/octet-stream"
                else:
                    raise InvalidArgumentException(
                        f"Unsupported file data type: {type(entry.data)}"
                    )

                yield f"--{boundary}\r\n".encode()
                yield b'Content-Disposition: form-data; name="metadata"; filename="metadata"\r\n'
                yield b"Content-Type: application/json\r\n\r\n"
                yield metadata_json.encode()
                yield b"\r\n"

                filename = _multipart_header_filename(os.path.basename(entry.path) or "file")
                yield f"--{boundary}\r\n".encode()
                yield (
                    f'Content-Disposition: form-data; name="file"; '
                    f'filename="{filename}"\r\n'
                ).encode()
                yield f"Content-Type: {content_type}\r\n\r\n".encode()

                if isinstance(content, bytes):
                    yield content
                else:
                    while True:
                        chunk = content.read(64 * 1024)
                        if not chunk:
                            break
                        if isinstance(chunk, str):
                            yield chunk.encode()
                        else:
                            yield chunk
                yield b"\r\n"

            yield f"--{boundary}--\r\n".encode()

        return await client.post(
            url,
            content=_body(),
            headers={"content-type": f"multipart/form-data; boundary={boundary}"},
            follow_redirects=False,
        )

    async def write_file(
        self,
        path: str,
        data: str | bytes | IOBase,
        *,
        encoding: str = "utf-8",
        mode: int = 755,
        owner: str | None = None,
        group: str | None = None,
    ) -> None:
        """Write single file (convenience method)."""
        entry = WriteEntry(
            path=path,
            data=data,
            mode=mode,
            owner=owner,
            group=group,
            encoding=encoding,
        )
        await self.write_files([entry])

    async def create_directories(self, entries: list[WriteEntry]) -> None:
        """Create multiple directories with specified permissions.

        Args:
            entries: List of directory entries with paths and permissions

        Raises:
            SandboxException: If directory creation fails
        """
        try:
            from opensandbox.api.execd.api.filesystem import make_dirs

            client = await self._get_client()
            response_obj = await make_dirs.asyncio_detailed(
                client=client,
                body=FilesystemModelConverter.to_api_make_dirs_body(entries),
            )

            handle_api_error(response_obj, "Create directories")

        except Exception as e:
            logger.error("Failed to create directories", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def delete_files(self, paths: list[str]) -> None:
        """Delete files using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import remove_files

            client = await self._get_client()
            response_obj = await remove_files.asyncio_detailed(
                client=client,
                path=paths,
            )

            handle_api_error(response_obj, "Delete files")

        except Exception as e:
            logger.error(f"Failed to delete {len(paths)} files", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def delete_directories(self, paths: list[str]) -> None:
        """Delete directories using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import remove_dirs

            client = await self._get_client()
            response_obj = await remove_dirs.asyncio_detailed(
                client=client,
                path=paths,
            )

            handle_api_error(response_obj, "Delete directories")

        except Exception as e:
            logger.error(f"Failed to delete {len(paths)} directories", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def move_files(self, entries: list[MoveEntry]) -> None:
        """Move or rename multiple files and directories.

        Args:
            entries: List of move operations with source and destination paths

        Raises:
            SandboxException: If move operations fail
        """
        try:
            from opensandbox.api.execd.api.filesystem import rename_files
            rename_items = FilesystemModelConverter.to_api_rename_file_items(entries)

            client = await self._get_client()
            response_obj = await rename_files.asyncio_detailed(
                client=client,
                body=rename_items,
            )

            handle_api_error(response_obj, "Move files")

        except Exception as e:
            logger.error("Failed to move files", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def set_permissions(self, entries: list[SetPermissionEntry]) -> None:
        """Set file permissions using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import chmod_files

            client = await self._get_client()
            response_obj = await chmod_files.asyncio_detailed(
                client=client,
                body=FilesystemModelConverter.to_api_chmod_files_body(entries),
            )

            handle_api_error(response_obj, "Set permissions")

        except Exception as e:
            logger.error("Failed to set permissions", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def replace_contents(self, entries: list[ContentReplaceEntry]) -> None:
        """Replace file contents using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import replace_content

            client = await self._get_client()
            response_obj = await replace_content.asyncio_detailed(
                client=client,
                body=FilesystemModelConverter.to_api_replace_content_body(entries),
            )
            handle_api_error(response_obj, "Replace contents")

        except Exception as e:
            logger.error("Failed to replace contents", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def replace_contents_detailed(self, entries: list[ContentReplaceEntry]) -> list[ContentReplaceResult]:
        """Replace file contents and return per-file replacement counts."""
        try:
            from opensandbox.api.execd.api.filesystem import replace_content

            client = await self._get_client()
            response_obj = await replace_content.asyncio_detailed(
                client=client,
                body=FilesystemModelConverter.to_api_replace_content_body(entries),
                verbose=True,
            )

            handle_api_error(response_obj, "Replace contents")

            return FilesystemModelConverter.to_replace_results(response_obj.parsed)

        except Exception as e:
            logger.error("Failed to replace contents", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def search(self, entry: SearchEntry) -> list[EntryInfo]:
        """Search files using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import search_files
            from opensandbox.api.execd.models import FileInfo

            client = await self._get_client()
            response_obj = await search_files.asyncio_detailed(
                client=client,
                path=entry.path,
                pattern=entry.pattern,
            )

            handle_api_error(response_obj, "Search files")

            parsed = response_obj.parsed
            if not parsed:
                return []

            if isinstance(parsed, list) and all(isinstance(x, FileInfo) for x in parsed):
                return FilesystemModelConverter.to_entry_info_list(parsed)
            raise SandboxApiException(
                message="Search files failed: unexpected response type",
                request_id=extract_request_id(getattr(response_obj, "headers", None)),
            )

        except Exception as e:
            logger.error("Failed to search files", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def list_directory(self, entry: DirectoryListEntry) -> list[EntryInfo]:
        """List directory contents using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import list_directory
            from opensandbox.api.execd.models import FileInfo
            from opensandbox.api.execd.types import UNSET

            client = await self._get_client()
            response_obj = await list_directory.asyncio_detailed(
                client=client,
                path=entry.path,
                depth=entry.depth if entry.depth is not None else UNSET,
            )

            handle_api_error(response_obj, "List directory")

            parsed = response_obj.parsed
            if not parsed:
                return []

            if isinstance(parsed, list) and all(isinstance(x, FileInfo) for x in parsed):
                return FilesystemModelConverter.to_entry_info_list(parsed)
            raise SandboxApiException(
                message="List directory failed: unexpected response type",
                request_id=extract_request_id(getattr(response_obj, "headers", None)),
            )

        except Exception as e:
            logger.error("Failed to list directory", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    async def get_file_info(self, paths: list[str]) -> dict[str, EntryInfo]:
        """Get file information using auto-generated API."""
        try:
            from opensandbox.api.execd.api.filesystem import get_files_info

            client = await self._get_client()
            response_obj = await get_files_info.asyncio_detailed(
                client=client,
                path=paths,
            )

            handle_api_error(response_obj, "Get file info")

            if not response_obj.parsed:
                return {}

            return FilesystemModelConverter.to_entry_info_map(response_obj.parsed)

        except Exception as e:
            logger.error(f"Failed to get file info for {len(paths)} paths", exc_info=e)
            raise ExceptionConverter.to_sandbox_exception(e) from e

    def _build_download_request(
            self,
            path: str,
            range_header: str | None = None,
            *,
            offset: int | None = None,
            limit: int | None = None,
    ) -> _DownloadRequest:
        """Build HTTP request for file download operations.

        Args:
            path: File path to download
            range_header: Optional range header for partial downloads
            offset: Starting line number (1-based) for line-based reading
            limit: Number of lines to return for line-based reading

        Returns:
            Dictionary containing URL, parameters, and headers for the request
        """
        url = self._get_execd_url(self.FILESYSTEM_DOWNLOAD_PATH)
        headers: dict[str, str] = {}
        params: dict[str, str] = {"path": path}

        if range_header:
            headers["Range"] = range_header

        if offset is not None:
            params["offset"] = str(offset)
        if limit is not None:
            params["limit"] = str(limit)

        return {
            "url": url,
            "params": params,
            "headers": headers,
        }