Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

7.9 KB
Raw Blame
#
# Copyright 2026 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
LRU + TTL endpoint cache with inflight deduplication.

Async version uses asyncio.Future for inflight dedup.
Sync version uses threading.Lock + threading.Event.
"""

from __future__ import annotations

import asyncio
import threading
import time
from collections import OrderedDict
from collections.abc import Awaitable, Callable

from opensandbox.models.sandboxes import SandboxEndpoint

DEFAULT_ENDPOINT_CACHE_TTL = 600.0
DEFAULT_ENDPOINT_CACHE_SIZE = 1024


class EndpointCache:
    """Thread-safe LRU+TTL endpoint cache for sync usage."""

    def __init__(
        self,
        maxsize: int = DEFAULT_ENDPOINT_CACHE_SIZE,
        ttl: float = DEFAULT_ENDPOINT_CACHE_TTL,
    ) -> None:
        self._maxsize = max(1, maxsize)
        self._ttl = ttl
        self._cache: OrderedDict[tuple, tuple[SandboxEndpoint, float]] = OrderedDict()
        self._lock = threading.Lock()
        self._inflight: dict[tuple, _InflightEntry] = {}
        self._generation = 0

    def get(self, key: tuple) -> SandboxEndpoint | None:
        with self._lock:
            entry = self._cache.get(key)
            if entry is None:
                return None
            endpoint, expires_at = entry
            if time.monotonic() >= expires_at:
                del self._cache[key]
                return None
            self._cache.move_to_end(key)
            return endpoint

    def put(self, key: tuple, endpoint: SandboxEndpoint) -> None:
        with self._lock:
            self._put_locked(key, endpoint)

    def _put_locked(self, key: tuple, endpoint: SandboxEndpoint) -> None:
        if key in self._cache:
            self._cache.move_to_end(key)
            self._cache[key] = (endpoint, time.monotonic() + self._ttl)
            return
        while len(self._cache) >= self._maxsize:
            self._cache.popitem(last=False)
        self._cache[key] = (endpoint, time.monotonic() + self._ttl)

    def invalidate(self, sandbox_id: str) -> None:
        with self._lock:
            self._generation += 1
            keys_to_remove = [k for k in self._cache if k[0] == sandbox_id]
            for k in keys_to_remove:
                del self._cache[k]
            inflight_to_remove = [k for k in self._inflight if k[0] == sandbox_id]
            for k in inflight_to_remove:
                del self._inflight[k]

    def get_or_fetch(
        self,
        key: tuple,
        fetch: Callable[[], SandboxEndpoint],
    ) -> SandboxEndpoint:
        with self._lock:
            entry = self._cache.get(key)
            if entry is not None:
                endpoint, expires_at = entry
                if time.monotonic() < expires_at:
                    self._cache.move_to_end(key)
                    return endpoint
                del self._cache[key]

            gen_before = self._generation
            if key in self._inflight:
                inf = self._inflight[key]
                is_owner = False
            else:
                inf = _InflightEntry()
                self._inflight[key] = inf
                is_owner = True

        if not is_owner:
            inf.event.wait()
            if inf.error is not None:
                raise inf.error
            return inf.result  # type: ignore[return-value]

        try:
            endpoint = fetch()
            with self._lock:
                if self._generation == gen_before:
                    self._put_locked(key, endpoint)
            inf.result = endpoint
            return endpoint
        except Exception as e:
            inf.error = e
            raise
        finally:
            with self._lock:
                if self._inflight.get(key) is inf:
                    self._inflight.pop(key)
            inf.event.set()


class _InflightEntry:
    __slots__ = ("event", "result", "error", "owner")

    def __init__(self) -> None:
        self.event = threading.Event()
        self.result: SandboxEndpoint | None = None
        self.error: Exception | None = None
        self.owner: bool = False


class _FetchCancelled(Exception):
    """Set on an inflight future when the caller fetching it is cancelled."""


class AsyncEndpointCache:
    """Async LRU+TTL endpoint cache with asyncio.Future-based inflight dedup."""

    def __init__(
        self,
        maxsize: int = DEFAULT_ENDPOINT_CACHE_SIZE,
        ttl: float = DEFAULT_ENDPOINT_CACHE_TTL,
    ) -> None:
        self._maxsize = max(1, maxsize)
        self._ttl = ttl
        self._cache: OrderedDict[tuple, tuple[SandboxEndpoint, float]] = OrderedDict()
        self._inflight: dict[tuple, asyncio.Future[SandboxEndpoint]] = {}
        self._generation = 0

    def get(self, key: tuple) -> SandboxEndpoint | None:
        entry = self._cache.get(key)
        if entry is None:
            return None
        endpoint, expires_at = entry
        if time.monotonic() >= expires_at:
            del self._cache[key]
            return None
        self._cache.move_to_end(key)
        return endpoint

    def put(self, key: tuple, endpoint: SandboxEndpoint) -> None:
        if key in self._cache:
            self._cache.move_to_end(key)
            self._cache[key] = (endpoint, time.monotonic() + self._ttl)
            return
        while len(self._cache) >= self._maxsize:
            self._cache.popitem(last=False)
        self._cache[key] = (endpoint, time.monotonic() + self._ttl)

    def invalidate(self, sandbox_id: str) -> None:
        self._generation += 1
        keys_to_remove = [k for k in self._cache if k[0] == sandbox_id]
        for k in keys_to_remove:
            del self._cache[k]
        inflight_to_remove = [k for k in self._inflight if k[0] == sandbox_id]
        for k in inflight_to_remove:
            del self._inflight[k]

    async def get_or_fetch(
        self,
        key: tuple,
        fetch: Callable[[], Awaitable[SandboxEndpoint]],
    ) -> SandboxEndpoint:
        while True:
            cached = self.get(key)
            if cached is not None:
                return cached

            inflight = self._inflight.get(key)
            if inflight is None:
                break
            try:
                return await asyncio.shield(inflight)
            except _FetchCancelled:
                # The caller that started the fetch was cancelled, not this one;
                # fetch again instead of raising its cancellation here.
                continue

        gen_before = self._generation
        loop = asyncio.get_running_loop()
        future: asyncio.Future[SandboxEndpoint] = loop.create_future()
        self._inflight[key] = future

        try:
            endpoint = await fetch()
            if self._generation == gen_before:
                self.put(key, endpoint)
            future.set_result(endpoint)
            return endpoint
        except BaseException as e:
            if not future.done():
                # Waiters must not inherit this caller's cancellation.
                future.set_exception(
                    _FetchCancelled() if isinstance(e, asyncio.CancelledError) else e
                )
                # The owner raises the original exception directly instead of awaiting
                # this future. Mark the exception as observed so asyncio does not log
                # "Future exception was never retrieved" when there are no waiters.
                future.exception()
            raise
        finally:
            if self._inflight.get(key) is future:
                self._inflight.pop(key)