Secure, Fast, and Extensible Sandbox runtime for AI agents.
#
# Copyright 2025 The OpenSandbox Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Synchronous sandbox pool implementation."""
from __future__ import annotations
import asyncio
import logging
import math
import threading
import time
from collections.abc import Callable
from concurrent.futures import CancelledError, Future, ThreadPoolExecutor
from datetime import timedelta
from typing import TypeVar
from opensandbox._pool_reconciler import ReconcileState, run_reconcile_tick
from opensandbox.config.connection_sync import ConnectionConfigSync
from opensandbox.exceptions import (
PoolAcquireFailedException,
PoolDestroyedException,
PoolEmptyException,
PoolNotRunningException,
PoolStateStoreUnavailableException,
SandboxReadyTimeoutException,
)
from opensandbox.internal.pool_tracing import (
WARMUP_COMMIT_SPAN,
WARMUP_CREATE_SPAN,
WARMUP_POST_PREPARE_CHECK_SPAN,
WARMUP_PREPARE_SPAN,
WARMUP_READINESS_CHECK_SPAN,
WARMUP_RENEW_SPAN,
PoolTracer,
annotate_health_span,
)
from opensandbox.internal.readiness import is_readiness_auth_error
from opensandbox.pool_types import (
AcquirePolicy,
IdleEntry,
PoolConfig,
PoolCreationSpec,
PoolDestroyState,
PooledSandboxCreateContext,
PooledSandboxCreateReason,
PooledSandboxCreator,
PoolLifecycleState,
PoolSnapshot,
PoolState,
_extra_create_kwargs,
effective_max_idle_attempts,
policy_falls_through_to_direct_create,
)
from opensandbox.pool_types import (
try_take_idle_with_min_ttl as _try_take_idle_with_min_ttl,
)
from opensandbox.sync.manager import SandboxManagerSync
from opensandbox.sync.sandbox import SandboxSync
from opensandbox.transport import RetryPolicy
from opensandbox.transport._sync_retry import RetrySyncTransport
logger = logging.getLogger(__name__)
_WARMUP_TERMINATION_TIMEOUT_SECONDS = 5.0
_RELEASE_ALL_IDLE_CONCURRENCY = 50
_RECONCILE_INTERVAL_SECONDS = 1.0
_CREATE_EXECUTOR_HEADROOM = 1.5
_T = TypeVar("_T")
class SandboxPoolSync:
"""Client-side synchronous sandbox pool aligned with Kotlin SandboxPool."""
def __init__(
self,
*,
pool_name: str,
max_idle: int,
state_store: object,
connection_config: ConnectionConfigSync,
creation_spec: PoolCreationSpec,
owner_id: str | None = None,
warmup_create_qps: int = 10,
warmup_concurrency: int = 128,
primary_lock_ttl: timedelta = timedelta(seconds=60),
degraded_threshold: int = 3,
acquire_ready_timeout: timedelta = timedelta(seconds=30),
acquire_health_check_polling_interval: timedelta = timedelta(milliseconds=200),
acquire_health_check: Callable[[SandboxSync], bool] | None = None,
acquire_skip_health_check: bool = False,
warmup_ready_timeout: timedelta = timedelta(seconds=30),
warmup_health_check_initial_delay: timedelta = timedelta(0),
warmup_health_check_polling_interval: timedelta = timedelta(milliseconds=500),
warmup_health_check: Callable[[SandboxSync], bool] | None = None,
warmup_sandbox_preparer: Callable[[SandboxSync], None] | None = None,
warmup_post_prepare_health_check: Callable[[SandboxSync], bool] | None = None,
warmup_post_prepare_health_check_timeout: timedelta = timedelta(seconds=30),
warmup_skip_health_check: bool = False,
idle_timeout: timedelta = timedelta(hours=24),
drain_timeout: timedelta = timedelta(seconds=30),
acquire_min_remaining_ttl: timedelta | None = None,
max_acquire_retries: int = 3,
sandbox_manager_factory: Callable[
[ConnectionConfigSync], SandboxManagerSync
] = SandboxManagerSync.create,
sandbox_factory: type[SandboxSync] = SandboxSync,
sandbox_creator: PooledSandboxCreator | None = None,
) -> None:
self._config = PoolConfig(
pool_name=pool_name,
owner_id=owner_id,
max_idle=max_idle,
warmup_create_qps=warmup_create_qps,
warmup_concurrency=warmup_concurrency,
primary_lock_ttl=primary_lock_ttl,
state_store=state_store, # type: ignore[arg-type]
connection_config=connection_config,
creation_spec=creation_spec,
degraded_threshold=degraded_threshold,
acquire_ready_timeout=acquire_ready_timeout,
acquire_health_check_polling_interval=acquire_health_check_polling_interval,
acquire_health_check=acquire_health_check,
acquire_skip_health_check=acquire_skip_health_check,
warmup_ready_timeout=warmup_ready_timeout,
warmup_health_check_initial_delay=warmup_health_check_initial_delay,
warmup_health_check_polling_interval=warmup_health_check_polling_interval,
warmup_health_check=warmup_health_check,
warmup_sandbox_preparer=warmup_sandbox_preparer,
warmup_post_prepare_health_check=warmup_post_prepare_health_check,
warmup_post_prepare_health_check_timeout=warmup_post_prepare_health_check_timeout,
warmup_skip_health_check=warmup_skip_health_check,
idle_timeout=idle_timeout,
drain_timeout=drain_timeout,
acquire_min_remaining_ttl=acquire_min_remaining_ttl,
sandbox_creator=sandbox_creator,
max_acquire_retries=max_acquire_retries,
)
self._state_store = self._config.state_store
self._connection_config = connection_config
self._creation_spec = creation_spec
self._sandbox_manager_factory = sandbox_manager_factory
self._sandbox_factory = sandbox_factory
self._pool_tracer = PoolTracer(connection_config.enable_tracing)
self._reconcile_state = ReconcileState(degraded_threshold)
self._current_max_idle = max_idle
self._lifecycle_state = PoolLifecycleState.NOT_STARTED
self._lifecycle_lock = threading.RLock()
self._stop_lock = threading.RLock()
self._reconcile_lock = threading.Lock()
self._in_flight = 0
self._in_flight_condition = threading.Condition()
self._stop_event = threading.Event()
self._scheduler_thread: threading.Thread | None = None
self._heartbeat_thread: threading.Thread | None = None
self._warmup_executor: ThreadPoolExecutor | None = None
self._create_executor: ThreadPoolExecutor | None = None
self._warmup_loop: asyncio.AbstractEventLoop | None = None
self._warmup_loop_thread: threading.Thread | None = None
self._warmup_futures: set[Future[None]] = set()
self._next_warmup_token = 0
# Maps each in-flight warmup token to the run generation that submitted
# it, so teardown can tell its own run's warmups from tokens left over
# by an earlier orphaned run.
self._warmup_tokens: dict[int, int] = {}
self._started_warmup_tokens: set[int] = set()
self._sandbox_manager: SandboxManagerSync | None = None
self._pool_connection_config: ConnectionConfigSync | None = None
self._pool_transport_owner: ConnectionConfigSync | None = None
self._run_generation = 0
self._leader_epoch = 0
self._primary_owned = False
self._accept_warmup_commits = False
self._warming_count = 0
self._warming_lock = threading.Lock()
def start(self) -> None:
# Hold _stop_lock across installation so a concurrent _stop_reconcile
# cannot capture a half-installed run (or tear down resources this run
# is about to install). The RLock lets this same thread re-enter the
# lock from the failure path below.
with self._stop_lock, self._lifecycle_lock:
if self._lifecycle_state in (
PoolLifecycleState.RUNNING,
PoolLifecycleState.STARTING,
):
return
self._lifecycle_state = PoolLifecycleState.STARTING
try:
# Reap resources a superseded shutdown() may have left behind
# (run_generation=None tears down unconditionally).
self._stop_reconcile(wait_for_warmup=False, run_generation=None)
self._ensure_pool_namespace_active()
self._warn_if_primary_lock_ttl_may_expire_during_warmup()
self._open_pool_transport()
self._sandbox_manager = self._create_sandbox_manager()
self._state_store.set_idle_entry_ttl(
self._config.pool_name, self._config.idle_timeout
)
self._state_store.set_max_idle(
self._config.pool_name, self._config.max_idle
)
self._run_generation += 1
self._primary_owned = False
self._accept_warmup_commits = True
self._create_executor = ThreadPoolExecutor(
max_workers=max(
1,
math.ceil(
self._config.warmup_create_qps * _CREATE_EXECUTOR_HEADROOM
),
),
thread_name_prefix=f"sandbox-pool-create-{self._config.pool_name}",
)
self._warmup_executor = ThreadPoolExecutor(
max_workers=self._config.warmup_concurrency,
thread_name_prefix=f"sandbox-pool-warmup-{self._config.pool_name}",
)
warmup_loop = asyncio.new_event_loop()
self._warmup_loop = warmup_loop
self._warmup_loop_thread = threading.Thread(
target=self._run_warmup_loop,
args=(warmup_loop,),
name=f"sandbox-pool-warmup-dispatch-{self._config.pool_name}",
daemon=True,
)
self._warmup_loop_thread.start()
stop_event = threading.Event()
self._stop_event = stop_event
self._scheduler_thread = threading.Thread(
target=self._run_scheduler,
args=(stop_event,),
name=f"sandbox-pool-reconcile-{self._config.pool_name}",
daemon=True,
)
self._lifecycle_state = PoolLifecycleState.RUNNING
self._scheduler_thread.start()
self._heartbeat_thread = threading.Thread(
target=self._run_heartbeat,
args=(stop_event, self._run_generation),
name=f"sandbox-pool-heartbeat-{self._config.pool_name}",
daemon=True,
)
self._heartbeat_thread.start()
except Exception:
self._stop_reconcile(
wait_for_warmup=True, run_generation=self._run_generation
)
self._close_provider()
self._lifecycle_state = PoolLifecycleState.STOPPED
raise
def acquire(
self,
sandbox_timeout: timedelta | None = None,
policy: AcquirePolicy = AcquirePolicy.DIRECT_CREATE,
) -> SandboxSync:
with self._lifecycle_lock:
if self._lifecycle_state != PoolLifecycleState.RUNNING:
state = self._lifecycle_state
self._raise_if_pool_namespace_destroyed()
raise PoolNotRunningException(
f"Cannot acquire when pool state is {state.value}"
)
operation_generation = self._run_generation
self._begin_operation()
try:
self._ensure_acquire_run_active(operation_generation)
self._ensure_pool_namespace_active_for_acquire(policy)
pool_name = self._config.pool_name
max_attempts = effective_max_idle_attempts(
policy, self._config.max_acquire_retries
)
# Accumulate discarded-alive sandbox ids across all take iterations so we schedule
# a single deferred cleanup, instead of one kill batch per retry.
pending_kill: list[str] = []
last_sandbox_id: str | None = None
last_idle_connect_failure: Exception | None = None
attempted_any = False
loop_exhausted = True
attempt = 0
while attempt < max_attempts:
self._ensure_acquire_run_active(operation_generation)
attempt += 1
try:
take_result = _try_take_idle_with_min_ttl(
self._state_store,
pool_name,
self._config.acquire_min_remaining_ttl,
)
except PoolStateStoreUnavailableException:
# State store outage. Per OSEP-0005, under policies that fall through to
# direct-create on empty idle we degrade to that fallback so the pool stays
# at least as available as raw SDK usage during store outages. Under
# non-fallthrough policies (FAIL_FAST / RETRY_NEXT_IDLE) we surface the
# outage as-is so callers can react.
if not policy_falls_through_to_direct_create(policy):
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
raise
logger.warning(
"acquire: state store unavailable, falling through to direct create "
"per policy=%s",
policy.value,
)
loop_exhausted = False
break
if take_result.discarded_alive_sandbox_ids:
pending_kill.extend(take_result.discarded_alive_sandbox_ids)
sandbox_id = take_result.sandbox_id
if sandbox_id is None:
# Idle buffer drained mid-loop (or was empty from the start). Stop retrying —
# another take round-trip is pure overhead.
loop_exhausted = False
break
last_sandbox_id = sandbox_id
attempted_any = True
try:
sandbox = self._sandbox_factory.connect(
sandbox_id,
connection_config=self._connection_for_pool_resource(),
health_check=self._config.acquire_health_check,
connect_timeout=self._config.acquire_ready_timeout,
health_check_polling_interval=(
self._config.acquire_health_check_polling_interval
),
skip_health_check=self._config.acquire_skip_health_check,
)
except AssertionError:
self._schedule_kill_discarded_alive(
pool_name, (*pending_kill, sandbox_id), source="acquire"
)
raise
except PoolDestroyedException:
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
raise
except Exception as exc:
# Auth/permission verdicts (401/403) cannot be fixed by
# re-probing, so surface the original error instead of burning
# retries. The taken candidate still needs a disposition, though:
# try_take already removed it from the store, and leaving it alive
# would leak it untracked (the renew-failure path below documents
# the same trap). Kill this one candidate together with the
# already-condemned ones; warmup replaces the slot. A per-sandbox
# token failure (direct-mode X-EXECD-ACCESS-TOKEN after execd
# restart / snapshot resume) is also cleaned up this way.
if is_readiness_auth_error(exc):
self._schedule_kill_discarded_alive(
pool_name, (*pending_kill, sandbox_id), source="acquire"
)
raise
# Connect / readiness / health-check failure — the idle candidate itself
# is unusable. Remove it, fire-and-forget the remote kill on the warmup
# executor so a slow DELETE (up to the lifecycle client's request_timeout,
# 30s by default) does not block the next retry iteration.
last_idle_connect_failure = exc
self._state_store.remove_idle(pool_name, sandbox_id)
self._schedule_kill_discarded_alive(
pool_name, (sandbox_id,), source="acquire-stale"
)
try:
self._ensure_acquire_run_active(operation_generation)
except PoolNotRunningException as retired:
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
raise retired from exc
self._ensure_pool_namespace_active()
continue
# Connect + readiness succeeded. From here on the sandbox is a healthy,
# borrowable idle: any failure below (renew rejection, namespace fenced) is
# NOT a candidate-specific problem, so we must not treat it as "stale idle"
# and burn another retry. Dispose the sandbox and surface the error.
try:
if sandbox_timeout is not None:
sandbox.renew(sandbox_timeout)
self._ensure_pool_namespace_active_after_create(sandbox)
self._ensure_acquire_run_active(operation_generation)
except PoolDestroyedException:
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
raise
except Exception as exc:
# Renew failed against a healthy sandbox. try_take_idle already popped this
# id out of the store; a bare close() would only release local HTTP resources
# and leave the remote sandbox alive-but-untracked until its server-side TTL
# expires. Kill it best-effort, then close local resources and re-raise.
logger.warning(
"Acquire renew failed after idle connect; killing remote sandbox and "
"not retrying (renew errors are not candidate-specific): "
"pool_name=%s sandbox_id=%s policy=%s error=%s",
pool_name,
sandbox_id,
policy.value,
exc,
)
try:
sandbox.kill()
except Exception as kill_exc:
logger.warning(
"Best-effort kill after renew failure failed: "
"pool_name=%s sandbox_id=%s error=%s",
pool_name,
sandbox_id,
kill_exc,
)
try:
sandbox.close()
except Exception as close_exc:
# Best-effort local resource release; original renew error must be the
# one that surfaces, so log at debug and continue with the raise below.
logger.debug(
"Best-effort close after renew failure failed: "
"pool_name=%s sandbox_id=%s error=%s",
pool_name,
sandbox_id,
close_exc,
)
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
raise
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
return sandbox
# Reached end of loop without a successful acquire. Fire deferred cleanup so the
# discarded-alive sandboxes do not linger; both the raise and the direct-create
# fallthrough benefit from async cleanup instead of a synchronous kill.
self._schedule_kill_discarded_alive(
pool_name, tuple(pending_kill), source="acquire"
)
if not attempted_any:
reason = "idle buffer empty"
elif loop_exhausted:
reason = (
f"idle connect failed for {max_attempts} candidate(s); "
f"last sandbox_id={last_sandbox_id} (stale or unreachable)"
)
else:
reason = (
f"idle connect failed for sandbox_id={last_sandbox_id}; "
f"idle buffer drained before reaching max_acquire_retries={max_attempts}"
)
if not policy_falls_through_to_direct_create(policy):
if attempted_any:
raise PoolAcquireFailedException(
f"Cannot acquire: {reason}; policy is {policy.value}",
last_idle_connect_failure,
)
raise PoolEmptyException(
f"Cannot acquire: {reason}; policy is {policy.value}"
)
self._ensure_acquire_run_active(operation_generation)
try:
sandbox = self._direct_create(sandbox_timeout, policy=policy)
except (AssertionError, PoolDestroyedException):
raise
except Exception as exc:
try:
self._ensure_acquire_run_active(operation_generation)
except PoolNotRunningException as retired:
raise retired from exc
raise
try:
self._ensure_acquire_run_active(operation_generation)
except PoolNotRunningException:
self._cleanup_uncommitted_warmup(sandbox)
raise
return sandbox
finally:
self._end_operation()
def resize(self, max_idle: int) -> None:
if max_idle < 0:
raise ValueError("max_idle must be >= 0")
self._ensure_pool_namespace_active()
self._state_store.set_max_idle(self._config.pool_name, max_idle)
self._current_max_idle = max_idle
def release_all_idle(self) -> int:
pool_name = self._config.pool_name
count = 0
temporary_manager: SandboxManagerSync | None = None
try:
while True:
sandbox_id = self._state_store.try_take_idle(pool_name)
if sandbox_id is None:
break
count += 1
try:
manager = self._sandbox_manager or temporary_manager
if manager is None:
manager = self._create_sandbox_manager()
temporary_manager = manager
manager.kill_sandbox(sandbox_id)
except Exception as exc:
logger.warning(
f"release_all_idle: failed to kill sandbox: pool_name={pool_name} sandbox_id={sandbox_id} error={exc}"
)
finally:
if temporary_manager is not None:
temporary_manager.close()
return count
def release_all_idle_parallel(
self, max_workers: int = _RELEASE_ALL_IDLE_CONCURRENCY
) -> int:
if max_workers <= 0:
raise ValueError("max_workers must be positive")
pool_name = self._config.pool_name
sandbox_ids: list[str] = []
drain_error: Exception | None = None
temporary_manager: SandboxManagerSync | None = None
try:
while True:
try:
sandbox_id = self._state_store.try_take_idle(pool_name)
except Exception as exc:
drain_error = exc
break
if sandbox_id is None:
break
sandbox_ids.append(sandbox_id)
if sandbox_ids:
manager = self._sandbox_manager
if manager is None:
try:
manager = self._create_sandbox_manager()
temporary_manager = manager
except Exception as exc:
logger.warning(
f"release_all_idle_parallel: failed to create sandbox manager; draining idle ids without remote kill: pool_name={pool_name} error={exc}"
)
def kill(sandbox_id: str) -> None:
if manager is None:
return
try:
manager.kill_sandbox(sandbox_id)
except Exception as exc:
logger.warning(
f"release_all_idle_parallel: failed to kill sandbox: pool_name={pool_name} sandbox_id={sandbox_id} error={exc}"
)
with ThreadPoolExecutor(
max_workers=min(max_workers, len(sandbox_ids)),
thread_name_prefix="sandbox-pool-release",
) as executor:
list(executor.map(kill, sandbox_ids))
finally:
if temporary_manager is not None:
temporary_manager.close()
if drain_error is not None:
raise drain_error
return len(sandbox_ids)
def snapshot(self) -> PoolSnapshot:
lifecycle_state = self._lifecycle_state
if lifecycle_state in (
PoolLifecycleState.NOT_STARTED,
PoolLifecycleState.STOPPED,
):
state = PoolState.STOPPED
elif lifecycle_state == PoolLifecycleState.DRAINING:
state = PoolState.DRAINING
else:
state = self._reconcile_state.state
counters = self._state_store.snapshot_counters(self._config.pool_name)
return PoolSnapshot(
state=state,
lifecycle_state=lifecycle_state,
idle_count=counters.idle_count,
max_idle=self._resolve_max_idle(),
failure_count=self._reconcile_state.failure_count,
backoff_active=self._reconcile_state.is_backoff_active(),
last_error=self._reconcile_state.last_error,
in_flight_operations=self._in_flight,
)
def snapshot_idle_entries(self) -> list[IdleEntry]:
return self._state_store.snapshot_idle_entries(self._config.pool_name)
def shutdown(self, graceful: bool = True) -> None:
with self._lifecycle_lock:
if self._lifecycle_state in (
PoolLifecycleState.NOT_STARTED,
PoolLifecycleState.STOPPED,
):
self._lifecycle_state = PoolLifecycleState.STOPPED
return
run_generation = self._run_generation
if not graceful:
self._accept_warmup_commits = False
self._lifecycle_state = PoolLifecycleState.DRAINING
if not graceful:
self._stop_reconcile(wait_for_warmup=False, run_generation=run_generation)
with self._lifecycle_lock:
if self._run_generation == run_generation:
self._lifecycle_state = PoolLifecycleState.STOPPED
self._close_provider()
return
drained = self._await_in_flight_drain(self._config.drain_timeout)
if not drained:
logger.warning(
f"Pool graceful shutdown timed out waiting in-flight operations: pool_name={self._config.pool_name} in_flight={self._in_flight} timeout_ms={int(self._config.drain_timeout.total_seconds() * 1000)}"
)
with self._lifecycle_lock:
if self._run_generation == run_generation:
self._accept_warmup_commits = False
self._stop_reconcile(wait_for_warmup=False, run_generation=run_generation)
with self._lifecycle_lock:
if self._run_generation == run_generation:
self._lifecycle_state = PoolLifecycleState.STOPPED
self._close_provider()
def _run_scheduler(self, stop_event: threading.Event) -> None:
initial_delay = 0 if self._config.max_idle > 0 else _RECONCILE_INTERVAL_SECONDS
if initial_delay > 0 and stop_event.wait(initial_delay):
return
while not stop_event.is_set():
self._run_reconcile_tick()
if stop_event.wait(_RECONCILE_INTERVAL_SECONDS):
break
def _run_reconcile_tick(self) -> None:
if self._lifecycle_state != PoolLifecycleState.RUNNING:
return
with self._reconcile_lock:
self._run_reconcile_tick_locked()
def _run_reconcile_tick_locked(self) -> None:
if self._lifecycle_state != PoolLifecycleState.RUNNING:
return
self._begin_operation()
try:
if self._lifecycle_state != PoolLifecycleState.RUNNING:
return
if (
self._state_store.get_destroy_state(self._config.pool_name)
!= PoolDestroyState.ACTIVE
):
self._stop_after_pool_namespace_destroyed()
return
with self._warming_lock:
warming_count = self._warming_count
primary_owned = run_reconcile_tick(
config=self._config.with_max_idle(self._resolve_max_idle()),
state_store=self._state_store,
on_discard_sandbox=self._discard_sandbox_callback,
warming_count=warming_count,
submit_warmups=self._submit_warmups,
on_primary_acquired=self._mark_primary_acquired,
)
if not primary_owned:
self._mark_primary_lost()
except Exception as exc:
self._mark_primary_lost()
logger.error(
f"Pool reconcile tick failed unexpectedly: pool_name={self._config.pool_name}",
exc_info=exc,
)
finally:
self._end_operation()
def _submit_warmups(self, count: int) -> None:
loop = self._warmup_loop
if loop is None:
return
generation = self._run_generation
leader_epoch = self._leader_epoch
for _ in range(count):
self._begin_operation()
with self._warming_lock:
self._warming_count += 1
self._next_warmup_token += 1
token = self._next_warmup_token
self._warmup_tokens[token] = generation
coroutine = self._run_warmup_async(
generation, leader_epoch, token, time.time_ns()
)
try:
future = asyncio.run_coroutine_threadsafe(coroutine, loop)
with self._warming_lock:
self._warmup_futures.add(future)
future.add_done_callback(
lambda done, current=token: self._warmup_done(done, current)
)
except BaseException as exc:
# ``run_coroutine_threadsafe`` does not take ownership when the
# dispatch loop has already stopped. Close the unsubmitted
# coroutine explicitly so shutdown races do not leak it.
coroutine.close()
self._complete_warmup(token, exc)
async def _run_warmup_async(
self,
generation: int,
leader_epoch: int,
token: int,
submitted_ns: int,
) -> None:
with self._warming_lock:
self._started_warmup_tokens.add(token)
create_executor = self._create_executor
warmup_executor = self._warmup_executor
if create_executor is None or warmup_executor is None:
self._complete_warmup(
token, RuntimeError("warmup stage executor is stopped")
)
return
# The executors are captured once because _stop_reconcile may null the
# instance attributes while this coroutine is still in flight (either
# tearing down this run or orphaning it when a warmup is stuck). Every
# stage below -- including the finally cleanup -- must keep using the
# executors this warmup started with so an uncommitted sandbox still
# gets terminated (issue #2056).
loop = asyncio.get_running_loop()
sandbox: SandboxSync | None = None
committed = False
stage = "create"
warmup_trace = self._pool_tracer.start_warmup(
pool_name=self._config.pool_name,
owner_id=str(self._config.owner_id),
run_generation=generation,
leader_epoch=leader_epoch,
submitted_ns=submitted_ns,
image=str(self._creation_spec.image),
)
try:
with warmup_trace.phase(WARMUP_CREATE_SPAN):
# Keep the underlying create future independently observable. A
# forced shutdown can retire the warmup loop before a slow control-
# plane create returns; in that case the late sandbox still needs
# to be terminated after this coroutine is gone.
create_future = create_executor.submit(self._build_warmup_sandbox)
try:
created_sandbox = await asyncio.shield(
asyncio.wrap_future(create_future, loop=loop)
)
except asyncio.CancelledError:
create_future.add_done_callback(
self._cleanup_cancelled_warmup_create
)
raise
sandbox = created_sandbox
warmup_trace.set_sandbox_id(sandbox.id)
await self._run_stage(
self._ensure_pool_namespace_active, executor=warmup_executor
)
readiness_deadline = (
loop.time() + self._config.warmup_ready_timeout.total_seconds()
)
if (
not self._config.warmup_skip_health_check
and self._config.warmup_health_check_initial_delay.total_seconds() > 0
):
await asyncio.sleep(
min(
self._config.warmup_health_check_initial_delay.total_seconds(),
self._config.warmup_ready_timeout.total_seconds(),
)
)
if not self._config.warmup_skip_health_check:
stage = "readiness"
with warmup_trace.phase(WARMUP_READINESS_CHECK_SPAN) as health_span:
await self._wait_until_healthy_async(
sandbox,
self._config.warmup_health_check,
readiness_deadline,
"warmup readiness",
health_span,
executor=warmup_executor,
)
preparer = self._config.warmup_sandbox_preparer
if preparer is not None:
stage = "prepare"
with warmup_trace.phase(WARMUP_PREPARE_SPAN):
await self._run_stage(
lambda: preparer(sandbox), executor=warmup_executor
)
post_check = self._config.warmup_post_prepare_health_check
if post_check is not None:
stage = "post_prepare_readiness"
with warmup_trace.phase(WARMUP_POST_PREPARE_CHECK_SPAN) as health_span:
await self._wait_until_healthy_async(
sandbox,
post_check,
loop.time()
+ self._config.warmup_post_prepare_health_check_timeout.total_seconds(),
"post-prepare readiness",
health_span,
executor=warmup_executor,
)
stage = "renew"
with warmup_trace.phase(WARMUP_RENEW_SPAN):
await self._run_stage(
lambda: sandbox.renew(self._config.idle_timeout),
executor=warmup_executor,
)
await self._run_stage(
lambda: self._ensure_pool_namespace_active_after_create(sandbox),
executor=warmup_executor,
)
stage = "commit"
with warmup_trace.phase(WARMUP_COMMIT_SPAN):
may_commit = await self._run_stage(
lambda: self._can_commit_warmup(generation, leader_epoch),
executor=warmup_executor,
)
if not may_commit:
warmup_trace.end_dropped(stage, "leadership_lost")
return
await self._run_stage(
lambda: self._state_store.put_idle(
self._config.pool_name, sandbox.id
),
executor=warmup_executor,
)
if not self._can_commit_locally(generation, leader_epoch):
await self._run_stage(
lambda: self._state_store.remove_idle(
self._config.pool_name, sandbox.id
),
executor=warmup_executor,
)
warmup_trace.end_dropped(stage, "run_retired")
return
committed = True
self._reconcile_state.record_success()
warmup_trace.end_success()
except asyncio.CancelledError:
warmup_trace.end_cancelled(stage)
raise
except BaseException as exc:
self._reconcile_state.record_failure(str(exc))
warmup_trace.end_failure(stage, exc)
logger.warning(
f"Pool warmup failed: pool_name={self._config.pool_name} error={exc}"
)
finally:
try:
if sandbox is not None:
if committed:
await self._run_stage(sandbox.close, executor=warmup_executor)
else:
await self._run_stage(
lambda: self._cleanup_uncommitted_warmup(sandbox),
executor=warmup_executor,
)
finally:
self._complete_warmup(token, None)
# If this run's warmup loop was orphaned by _stop_reconcile
# while this warmup was still in flight, stop the loop once
# nothing is left scheduled on it so its daemon thread can
# exit instead of parking until process exit.
loop.call_soon_threadsafe(self._maybe_stop_orphaned_warmup_loop, loop)
async def _run_stage(
self,
action: Callable[[], _T],
executor: ThreadPoolExecutor | None = None,
) -> _T:
target = executor or self._warmup_executor
if target is None:
raise RuntimeError("warmup stage executor is stopped")
future = asyncio.get_running_loop().run_in_executor(target, action)
try:
return await asyncio.shield(future)
except asyncio.CancelledError:
try:
await future
finally:
raise
def _maybe_stop_orphaned_warmup_loop(self, loop: asyncio.AbstractEventLoop) -> None:
# Runs on the warmup loop itself after a warmup task has finished. A
# loop the pool still owns is stopped by _stop_reconcile; an orphaned
# loop stops itself here once its last task is done.
if self._warmup_loop is loop:
return
if asyncio.all_tasks(loop):
return
loop.stop()
async def _wait_until_healthy_async(
self,
sandbox: SandboxSync,
health_check: Callable[[SandboxSync], bool] | None,
deadline: float,
stage: str,
trace_span: object = None,
executor: ThreadPoolExecutor | None = None,
) -> None:
last_error: Exception | None = None
loop = asyncio.get_running_loop()
attempt_count = 0
false_count = 0
exception_count = 0
while True:
try:
attempt_count += 1
healthy = await self._run_stage(
lambda: (
health_check(sandbox)
if health_check is not None
else sandbox.is_healthy()
),
executor=executor,
)
if healthy:
annotate_health_span(
trace_span, # type: ignore[arg-type]
attempt_count=attempt_count,
false_count=false_count,
exception_count=exception_count,
)
return
false_count += 1
last_error = None
except asyncio.CancelledError:
raise
except Exception as exc:
exception_count += 1
last_error = exc
remaining = deadline - loop.time()
if remaining <= 0:
annotate_health_span(
trace_span, # type: ignore[arg-type]
attempt_count=attempt_count,
false_count=false_count,
exception_count=exception_count,
)
raise SandboxReadyTimeoutException(
f"Pool {stage} timed out", cause=last_error
)
await asyncio.sleep(
min(
remaining,
self._config.warmup_health_check_polling_interval.total_seconds(),
)
)
@staticmethod
def _run_warmup_loop(loop: asyncio.AbstractEventLoop) -> None:
asyncio.set_event_loop(loop)
loop.run_forever()
def _warmup_done(self, future: Future[None], token: int) -> None:
with self._warming_lock:
self._warmup_futures.discard(future)
started = token in self._started_warmup_tokens
if not started:
error: BaseException | None = (
CancelledError() if future.cancelled() else future.exception()
)
self._complete_warmup(token, error)
if future.cancelled():
return
error = future.exception()
if error is not None:
logger.warning(
f"Pool warmup task failed: pool_name={self._config.pool_name} error={error}"
)
def _complete_warmup(self, token: int, error: BaseException | None) -> None:
with self._warming_lock:
if token not in self._warmup_tokens:
return
del self._warmup_tokens[token]
self._started_warmup_tokens.discard(token)
self._warming_count = max(0, self._warming_count - 1)
if error is not None and not isinstance(error, CancelledError):
self._reconcile_state.record_failure(str(error))
logger.warning(
f"Pool warmup create failed: pool_name={self._config.pool_name} error={error}"
)
self._end_operation()
def _can_commit_warmup(self, generation: int, leader_epoch: int) -> bool:
if not self._can_commit_locally(generation, leader_epoch):
return False
renewed = self._state_store.renew_primary_lock(
self._config.pool_name,
str(self._config.owner_id),
self._config.primary_lock_ttl,
)
if not renewed:
self._mark_primary_lost()
return False
return self._can_commit_locally(generation, leader_epoch)
def _can_commit_locally(self, generation: int, leader_epoch: int) -> bool:
with self._lifecycle_lock:
return (
generation == self._run_generation
and leader_epoch == self._leader_epoch
and self._primary_owned
and self._accept_warmup_commits
and self._lifecycle_state
in (PoolLifecycleState.RUNNING, PoolLifecycleState.DRAINING)
)
def _mark_primary_acquired(self) -> None:
with self._lifecycle_lock:
if self._primary_owned:
return
self._primary_owned = True
self._leader_epoch += 1
def _mark_primary_lost(self) -> None:
futures: tuple[Future[None], ...] = ()
with self._lifecycle_lock:
if not self._primary_owned:
return
self._primary_owned = False
self._leader_epoch += 1
with self._warming_lock:
futures = tuple(self._warmup_futures)
for future in futures:
future.cancel()
def _run_heartbeat(self, stop_event: threading.Event, generation: int) -> None:
interval = max(
0.001,
min(1.0, self._config.primary_lock_ttl.total_seconds() / 3),
)
while not stop_event.wait(interval):
with self._lifecycle_lock:
if generation != self._run_generation:
return
should_renew = self._primary_owned and self._lifecycle_state in (
PoolLifecycleState.RUNNING,
PoolLifecycleState.DRAINING,
)
if not should_renew:
continue
try:
renewed = self._state_store.renew_primary_lock(
self._config.pool_name,
str(self._config.owner_id),
self._config.primary_lock_ttl,
)
if not renewed:
self._mark_primary_lost()
except Exception as exc:
logger.warning(
f"Pool primary heartbeat failed: pool_name={self._config.pool_name} error={exc}"
)
def _cleanup_uncommitted_warmup(self, sandbox: SandboxSync) -> None:
try:
sandbox.kill()
except Exception:
pass
finally:
try:
sandbox.close()
except Exception:
pass
def _cleanup_cancelled_warmup_create(
self, future: Future[SandboxSync]
) -> None:
"""Terminate a sandbox whose create completed after forced shutdown.
This callback runs on the create worker, independently of the retired
warmup event loop. It intentionally uses a fresh manager/transport because
the pool-owned transport may already be closed by shutdown.
"""
try:
sandbox = future.result()
except BaseException:
return
manager: SandboxManagerSync | None = None
try:
base = self._connection_config.model_copy(
update={
"transport": None,
"retry_policy": RetryPolicy.disabled(),
}
)
cleanup_config = base.with_transport_if_missing(
max_connections=1,
max_keepalive_connections=1,
)
manager = self._sandbox_manager_factory(cleanup_config)
manager.kill_sandbox(sandbox.id)
except Exception as exc:
logger.warning(
f"Pool late warmup cleanup failed: pool_name={self._config.pool_name} sandbox_id={sandbox.id} error={exc}"
)
finally:
if manager is not None:
try:
manager.close()
except Exception:
pass
try:
sandbox.close()
except Exception:
pass
def _build_warmup_sandbox(self) -> SandboxSync:
if self._config.sandbox_creator is not None:
return self._build_sandbox_from_creator(
creator=self._config.sandbox_creator,
reason=PooledSandboxCreateReason.WARMUP,
ready_timeout=self._config.warmup_ready_timeout,
health_check_polling_interval=self._config.warmup_health_check_polling_interval,
skip_health_check=True,
health_check=self._config.warmup_health_check,
)
spec = self._creation_spec
create_kwargs = _extra_create_kwargs(spec)
return self._sandbox_factory.create(
spec.image,
timeout=self._config.idle_timeout,
ready_timeout=self._config.warmup_ready_timeout,
env=spec.env,
metadata=spec.metadata,
resource=spec.resource,
network_policy=spec.network_policy,
**create_kwargs,
platform=spec.platform,
extensions=spec.extensions,
secure_access=spec.secure_access,
entrypoint=spec.entrypoint,
volumes=spec.volumes,
connection_config=self._connection_for_warmup_create(),
health_check=self._config.warmup_health_check,
health_check_polling_interval=self._config.warmup_health_check_polling_interval,
skip_health_check=True,
)
def _direct_create(
self,
sandbox_timeout: timedelta | None,
policy: AcquirePolicy = AcquirePolicy.DIRECT_CREATE,
) -> SandboxSync:
# policy-aware namespace check: if the state store is down and the policy is a
# fallthrough one, treat destroy-state as unknown and proceed to direct-create
# instead of surfacing the outage. See _ensure_pool_namespace_active_for_acquire
# for the full rationale.
self._ensure_pool_namespace_active_for_acquire(policy)
if self._config.sandbox_creator is not None:
sandbox = self._build_sandbox_from_creator(
creator=self._config.sandbox_creator,
reason=PooledSandboxCreateReason.DIRECT_CREATE,
ready_timeout=self._config.acquire_ready_timeout,
health_check_polling_interval=self._config.acquire_health_check_polling_interval,
skip_health_check=self._config.acquire_skip_health_check,
health_check=self._config.acquire_health_check,
)
if sandbox_timeout is not None:
try:
sandbox.renew(sandbox_timeout)
except BaseException:
try:
sandbox.kill()
finally:
sandbox.close()
raise
self._ensure_pool_namespace_active_after_create(sandbox, policy=policy)
return sandbox
spec = self._creation_spec
create_kwargs = _extra_create_kwargs(spec)
sandbox = self._sandbox_factory.create(
spec.image,
timeout=self._config.idle_timeout,
ready_timeout=self._config.acquire_ready_timeout,
env=spec.env,
metadata=spec.metadata,
resource=spec.resource,
network_policy=spec.network_policy,
**create_kwargs,
platform=spec.platform,
extensions=spec.extensions,
secure_access=spec.secure_access,
entrypoint=spec.entrypoint,
volumes=spec.volumes,
connection_config=self._connection_for_pool_resource(),
health_check=self._config.acquire_health_check,
health_check_polling_interval=self._config.acquire_health_check_polling_interval,
skip_health_check=self._config.acquire_skip_health_check,
)
if sandbox_timeout is not None:
try:
sandbox.renew(sandbox_timeout)
except BaseException:
try:
sandbox.kill()
finally:
sandbox.close()
raise
self._ensure_pool_namespace_active_after_create(sandbox, policy=policy)
return sandbox
def _ensure_pool_namespace_active(self) -> None:
state = self._state_store.get_destroy_state(self._config.pool_name)
if state != PoolDestroyState.ACTIVE:
raise PoolDestroyedException(
f"Pool namespace is {state.value}: pool_name={self._config.pool_name}"
)
def _ensure_acquire_run_active(self, generation: int) -> None:
with self._lifecycle_lock:
if (
generation == self._run_generation
and self._lifecycle_state == PoolLifecycleState.RUNNING
):
return
state = self._lifecycle_state
self._raise_if_pool_namespace_destroyed()
raise PoolNotRunningException(
"Cannot acquire from a retired pool run: "
f"pool_name={self._config.pool_name} state={state.value}"
)
def _ensure_pool_namespace_active_for_acquire(self, policy: AcquirePolicy) -> None:
"""Namespace-active check on the acquire path with graceful degradation.
Same as :meth:`_ensure_pool_namespace_active`, but when the state store itself
is unavailable (``PoolStateStoreUnavailableException``) and the effective
``policy`` falls through to direct-create on empty idle, we treat the destroy
state as *unknown* and allow the acquire to proceed. This is the necessary
counterpart to the state-store-outage fallthrough already implemented at the
``try_take_idle`` and ``_direct_create`` call sites (see OSEP-0005 error-code
matrix): without it a full Redis outage would short-circuit acquire before the
fallthrough branch could run, making ``RETRY_NEXT_IDLE_THEN_CREATE`` and
``DIRECT_CREATE`` less available than documented.
Fail-closed behavior for non-fallthrough policies (``FAIL_FAST`` /
``RETRY_NEXT_IDLE``) is preserved: the outage is surfaced as-is so callers
can react.
"""
try:
self._ensure_pool_namespace_active()
except PoolStateStoreUnavailableException:
if not policy_falls_through_to_direct_create(policy):
raise
logger.warning(
"acquire: state store unavailable during namespace check, "
"assuming ACTIVE and degrading to direct-create per policy=%s",
policy.value,
)
def _raise_if_pool_namespace_destroyed(self) -> None:
try:
self._ensure_pool_namespace_active()
except PoolDestroyedException:
raise
except Exception:
return
def _ensure_pool_namespace_active_after_create(
self,
sandbox: SandboxSync,
policy: AcquirePolicy | None = None,
) -> None:
"""Post-create fence check.
If the state store itself is unavailable we cannot tell whether the pool was
destroyed, so under a fallthrough ``policy`` we assume ACTIVE and keep the
freshly-created sandbox (mirrors the OSEP-0005 acquire-outage semantics).
Non-fallthrough policies keep the original fail-closed behavior for backward
compatibility.
"""
try:
self._ensure_pool_namespace_active()
except PoolStateStoreUnavailableException:
if policy is not None and policy_falls_through_to_direct_create(policy):
logger.warning(
"acquire: state store unavailable during post-create fence check, "
"keeping sandbox and degrading per policy=%s sandbox_id=%s",
policy.value,
sandbox.id,
)
return
try:
sandbox.kill()
except Exception as exc:
logger.warning(
"Pool sandbox cleanup after store-outage fence failed: pool_name=%s "
"sandbox_id=%s operation=kill error=%s",
self._config.pool_name,
sandbox.id,
exc,
)
try:
sandbox.close()
except Exception as exc:
logger.warning(
"Pool sandbox cleanup after store-outage fence failed: pool_name=%s "
"sandbox_id=%s operation=close error=%s",
self._config.pool_name,
sandbox.id,
exc,
)
raise
except BaseException:
try:
sandbox.kill()
except Exception as exc:
logger.warning(
"Pool sandbox cleanup after fence failed: pool_name=%s "
"sandbox_id=%s operation=kill error=%s",
self._config.pool_name,
sandbox.id,
exc,
)
try:
sandbox.close()
except Exception as exc:
logger.warning(
"Pool sandbox cleanup after fence failed: pool_name=%s "
"sandbox_id=%s operation=close error=%s",
self._config.pool_name,
sandbox.id,
exc,
)
raise
def _stop_after_pool_namespace_destroyed(self) -> None:
with self._lifecycle_lock:
if self._lifecycle_state == PoolLifecycleState.STOPPED:
return
run_generation = self._run_generation
self._accept_warmup_commits = False
self._lifecycle_state = PoolLifecycleState.STOPPED
self._stop_reconcile(
wait_for_warmup=False, join_scheduler=False, run_generation=run_generation
)
with self._lifecycle_lock:
if self._run_generation == run_generation:
self._close_provider()
def _build_sandbox_from_creator(
self,
*,
creator: PooledSandboxCreator,
reason: PooledSandboxCreateReason,
ready_timeout: timedelta,
health_check_polling_interval: timedelta,
skip_health_check: bool,
health_check: Callable[[SandboxSync], bool] | None,
) -> SandboxSync:
context = PooledSandboxCreateContext(
pool_name=self._config.pool_name,
owner_id=str(self._config.owner_id),
idle_timeout=self._config.idle_timeout,
reason=reason,
ready_timeout=ready_timeout,
health_check_polling_interval=health_check_polling_interval,
skip_health_check=skip_health_check,
health_check=health_check,
connection_config=(
self._connection_for_warmup_create()
if reason == PooledSandboxCreateReason.WARMUP
else self._connection_for_pool_resource()
),
)
return creator(context)
def _resolve_max_idle(self) -> int:
shared = self._state_store.get_max_idle(self._config.pool_name)
return self._current_max_idle if shared is None else shared
def _create_sandbox_manager(self) -> SandboxManagerSync:
return self._sandbox_manager_factory(self._connection_for_pool_resource())
def _connection_for_pool_resource(self) -> ConnectionConfigSync:
shared = self._pool_connection_config
if shared is None or self._pool_transport_owner is None:
return shared or self._connection_config
transport = shared.transport
if (
transport is None
or not self._connection_config.retry_policy.wraps_transport()
):
return shared
wrapped = RetrySyncTransport(
transport, self._connection_config.retry_policy, owns_inner=False
)
config = self._connection_config.model_copy(update={"transport": wrapped})
config._owns_transport = True
return config
def _connection_for_warmup_create(self) -> ConnectionConfigSync:
if self._pool_transport_owner is None:
base = self._pool_connection_config or self._connection_config
return base.model_copy(update={"disable_metrics": True})
# Staged-warmup sandboxes own short-lived transports. Keeping their
# mutually incompatible endpoint origins out of the pool-wide shared
# transport prevents lifecycle starvation and connection churn.
config = self._connection_config.model_copy(
update={
"transport": None,
"retry_policy": RetryPolicy.disabled(),
"disable_metrics": True,
}
)
config._owns_transport = True
return config
def _open_pool_transport(self) -> None:
if self._connection_config.transport is not None:
self._pool_connection_config = self._connection_config
self._pool_transport_owner = None
return
size = max(1, self._config.warmup_concurrency)
base = self._connection_config.model_copy(
update={"retry_policy": RetryPolicy.disabled()}
)
owner = base.with_transport_if_missing(
# warmup_concurrency maps to retained idle capacity, matching
# Kotlin's ConnectionPool; it must not cap active acquire calls.
max_connections=None,
max_keepalive_connections=size,
keepalive_expiry=300.0,
)
shared = self._connection_config.model_copy(
update={
"transport": owner.transport,
"retry_policy": RetryPolicy.disabled(),
}
)
shared._owns_transport = False
self._pool_transport_owner = owner
self._pool_connection_config = shared
def _discard_sandbox_callback(self, sandbox_id: str) -> None:
"""``Callable[[str], None]`` adapter for the reconciler's ``on_discard_sandbox``
hook. The reconciler does not care whether the kill succeeded — it only needs the
sandbox to be removed from the pool's bookkeeping — so we drop the bool return
value here.
"""
self._kill_sandbox_best_effort(sandbox_id)
def _kill_sandbox_best_effort(self, sandbox_id: str) -> bool:
"""Best-effort kill a sandbox via the pool's manager.
Returns ``True`` on a confirmed kill, ``False`` if no manager is available or the
kill raised. Failures are logged at WARNING and swallowed so the caller's primary
outcome is unaffected.
"""
if self._sandbox_manager is None:
return False
try:
self._sandbox_manager.kill_sandbox(sandbox_id)
return True
except Exception as exc:
logger.warning(
f"Pool sandbox cleanup failed: pool_name={self._config.pool_name} sandbox_id={sandbox_id} error={exc}"
)
return False
def _schedule_kill_discarded_alive(
self,
pool_name: str,
sandbox_ids: tuple[str, ...],
source: str,
) -> None:
"""Offload :meth:`_kill_discarded_alive` to the warmup executor so the caller does not
block on the kill RPCs. Falls back to inline execution when no executor is available
(e.g. mid-shutdown) — better to slow the caller than to drop the cleanup entirely.
"""
if not sandbox_ids:
return
executor = self._warmup_executor
if executor is None:
self._kill_discarded_alive(pool_name, sandbox_ids, source)
return
try:
executor.submit(self._kill_discarded_alive, pool_name, sandbox_ids, source)
except Exception as exc:
logger.debug(
f"Discarded-alive kill submit rejected, running inline: pool_name={pool_name} count={len(sandbox_ids)} error={exc}"
)
self._kill_discarded_alive(pool_name, sandbox_ids, source)
def _kill_discarded_alive(
self,
pool_name: str,
sandbox_ids: tuple[str, ...],
source: str,
) -> None:
"""Best-effort terminate sandboxes the store dropped because their remaining TTL
fell below ``acquire_min_remaining_ttl``. Without this, alive-but-near-expiry
sandboxes would linger past their pool membership until server-side TTL elapses.
"""
if not sandbox_ids:
return
for sandbox_id in sandbox_ids:
if self._kill_sandbox_best_effort(sandbox_id):
logger.debug(
f"Killed near-expiry idle sandbox: pool_name={pool_name} sandbox_id={sandbox_id} source={source}"
)
def _begin_operation(self) -> None:
with self._in_flight_condition:
self._in_flight += 1
def _end_operation(self) -> None:
with self._in_flight_condition:
self._in_flight -= 1
if self._in_flight <= 0:
self._in_flight = 0
self._in_flight_condition.notify_all()
def _await_in_flight_drain(self, timeout: timedelta) -> bool:
deadline = time.monotonic() + timeout.total_seconds()
with self._in_flight_condition:
while self._in_flight > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._in_flight_condition.wait(remaining)
return True
def _stop_reconcile(
self,
*,
wait_for_warmup: bool,
join_scheduler: bool = True,
run_generation: int | None = None,
) -> None:
# Serialize teardown so concurrent stop paths (shutdown() racing
# shutdown(), or start()'s failure path racing shutdown()) cannot
# capture-and-close the same warmup loop or executors twice, and so
# start() (which holds _stop_lock while installing) can never have a
# run torn down half-installed. Every wait inside the lock is bounded
# by a timeout: warmup commit-stage workers may be blocked on
# _lifecycle_lock while a failing start() holds it, so an unbounded
# executor wait here would deadlock the failing start for good.
# run_generation=None tears down unconditionally (start()'s pre-clean);
# an int skips teardown when a newer run already owns the resources.
with self._stop_lock:
if run_generation is not None and self._run_generation != run_generation:
return
self._stop_event.set()
thread = self._scheduler_thread
if (
join_scheduler
and thread is not None
and thread is not threading.current_thread()
):
thread.join(timeout=5)
if join_scheduler:
self._scheduler_thread = None
heartbeat = self._heartbeat_thread
if heartbeat is not None and heartbeat is not threading.current_thread():
heartbeat.join(timeout=5)
self._heartbeat_thread = None
with self._warming_lock:
warmup_futures = tuple(self._warmup_futures)
generation = self._run_generation
if not wait_for_warmup:
for future in warmup_futures:
future.cancel()
deadline = time.monotonic() + _WARMUP_TERMINATION_TIMEOUT_SECONDS
# Cancelling the thread-safe Future marks that wrapper done before the
# coroutine's shielded cleanup has finished. Keep the dispatch loop
# alive until every warmup token of this run reaches its terminal
# callback.
while time.monotonic() < deadline:
with self._warming_lock:
if not any(
token_generation == generation
for token_generation in self._warmup_tokens.values()
):
break
time.sleep(0.01)
for future in warmup_futures:
try:
future.result(timeout=max(0.0, deadline - time.monotonic()))
except BaseException:
pass
with self._warming_lock:
loop = self._warmup_loop
loop_thread = self._warmup_loop_thread
create_executor = self._create_executor
executor = self._warmup_executor
outstanding_tokens = frozenset(
token
for token, token_generation in self._warmup_tokens.items()
if token_generation == generation
)
self._warmup_loop = None
self._warmup_loop_thread = None
self._create_executor = None
self._warmup_executor = None
if loop is not None and outstanding_tokens:
# issue #2056: a warmup coroutine of this run is still genuinely
# in flight once the bounded wait above gives up -- _run_stage's
# own cancellation handling deliberately blocks on its underlying
# executor call instead of abandoning it. Stopping or closing the
# loop here would destroy that still-pending asyncio Task ("Task
# was destroyed but it is pending!") and skip the warmup's own
# sandbox cleanup in its finally block. Orphan this run's warmup
# resources instead: the coroutine captured them at start, so it
# can finish and clean up after itself, and the loop stops itself
# once nothing is left scheduled on it. The loop thread is a
# daemon, and start() installs fresh resources for a new run.
logger.warning(
f"Pool shutdown leaving warmup loop running for in-flight warmup: pool_name={self._config.pool_name} outstanding_warmups={len(outstanding_tokens)}"
)
elif loop is not None:
loop.call_soon_threadsafe(loop.stop)
if (
loop_thread is not None
and loop_thread is not threading.current_thread()
):
loop_thread.join(timeout=5)
if loop is not None and not loop.is_running():
loop.close()
if create_executor is not None:
create_executor.shutdown(wait=False, cancel_futures=True)
self._await_executor_threads(
create_executor, _WARMUP_TERMINATION_TIMEOUT_SECONDS
)
if executor is not None:
executor.shutdown(wait=False, cancel_futures=True)
self._await_executor_threads(
executor, _WARMUP_TERMINATION_TIMEOUT_SECONDS
)
# else: refs are all None -- an earlier call already reclaimed this
# run's resources (or orphaned them). Idempotent bookkeeping. Kept
# inside _stop_lock: start() holds _stop_lock and _lifecycle_lock
# together, so a thread inside this section can always take
# _lifecycle_lock without inverting the order, and start()'s failure
# path re-enters this lock on the same thread.
self._release_primary_lock_best_effort()
# Idempotent bookkeeping. Kept inside _stop_lock: start() holds
# _stop_lock and _lifecycle_lock together, so a thread inside this
# section can always take _lifecycle_lock without inverting the
# order, and start()'s failure path re-enters this lock on the
# same thread.
self._release_primary_lock_best_effort()
self._mark_primary_lost()
def _await_executor_threads(
self, executor: ThreadPoolExecutor, timeout_seconds: float
) -> None:
deadline = time.monotonic() + timeout_seconds
threads = list(getattr(executor, "_threads", ()))
for thread in threads:
remaining = deadline - time.monotonic()
if remaining <= 0:
return
thread.join(timeout=remaining)
def _release_primary_lock_best_effort(self) -> None:
try:
self._state_store.release_primary_lock(
self._config.pool_name, str(self._config.owner_id)
)
except Exception as exc:
logger.warning(
f"Pool primary lock release failed: pool_name={self._config.pool_name} owner_id={self._config.owner_id} error={exc}"
)
def _close_provider(self) -> None:
if self._sandbox_manager is not None:
self._sandbox_manager.close()
self._sandbox_manager = None
if self._pool_transport_owner is not None:
self._pool_transport_owner.close_transport_if_owned()
self._pool_transport_owner = None
self._pool_connection_config = None
def _warn_if_primary_lock_ttl_may_expire_during_warmup(self) -> None:
if self._config.primary_lock_ttl > self._config.warmup_ready_timeout:
return
logger.warning(
f"Pool primary lock TTL may expire during warmup: pool_name={self._config.pool_name} primary_lock_ttl_ms={int(self._config.primary_lock_ttl.total_seconds() * 1000)} warmup_ready_timeout_ms={int(self._config.warmup_ready_timeout.total_seconds() * 1000)}"
)