Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

43.6 KB
Raw Blame
// Copyright 2026 The OpenSandbox Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import { ConnectionConfig } from "./config/connection.js";
import {
  PoolAcquireFailedException,
  PoolDestroyedException,
  PoolEmptyException,
  PoolNotRunningException,
  PoolStateStoreUnavailableException,
  SandboxReadyTimeoutException,
} from "./core/exceptions.js";
import { subscribeAbort } from "./internal/abort.js";
import { ReadinessBudget } from "./internal/readiness.js";
import { PoolTracer, POOL_WARMUP_SPANS } from "./internal/poolTracing.js";
import { InMemoryPoolStateStore } from "./poolStore.js";
import {
  AcquirePolicy,
  PoolLifecycleState,
  PoolDestroyState,
  PoolState,
  PooledSandboxCreateReason,
  type IdleEntry,
  type PoolCreationSpec,
  type PoolHealthCheck,
  type PoolSnapshot,
  type PoolStateStore,
  type SandboxAcquireOptions,
  type SandboxPoolOptions,
  type TakeIdleResult,
} from "./poolTypes.js";
import { Sandbox } from "./sandbox.js";
import { SandboxManager } from "./manager.js";

const DEFAULT_IDLE_TIMEOUT_SECONDS = 24 * 60 * 60;
const DEFAULT_READY_TIMEOUT_SECONDS = 30;
const DEFAULT_POLLING_INTERVAL_MILLIS = 200;
const CREATE_CONCURRENCY_HEADROOM = 1.5;

interface ResolvedPoolOptions {
  poolName: string;
  maxIdle: number;
  connectionConfig: ConnectionConfig;
  creationSpec: PoolCreationSpec;
  sandboxCreator?: SandboxPoolOptions["sandboxCreator"];
  stateStore: PoolStateStore;
  ownerId: string;
  warmupCreateQps: number;
  warmupConcurrency: number;
  primaryLockTtlSeconds: number;
  degradedThreshold: number;
  idleTimeoutSeconds: number;
  drainTimeoutSeconds: number;
  acquireMinRemainingTtlSeconds: number;
  acquireReadyTimeoutSeconds: number;
  acquireHealthCheckPollingIntervalMillis: number;
  acquireHealthCheck?: PoolHealthCheck;
  acquireSkipHealthCheck: boolean;
  maxAcquireRetries: number;
  warmupReadyTimeoutSeconds: number;
  warmupHealthCheckInitialDelayMillis: number;
  warmupHealthCheckPollingIntervalMillis: number;
  warmupHealthCheck?: PoolHealthCheck;
  warmupSandboxPreparer?: SandboxPoolOptions["warmupSandboxPreparer"];
  warmupPostPrepareHealthCheck?: PoolHealthCheck;
  warmupPostPrepareHealthCheckTimeoutSeconds: number;
  warmupSkipHealthCheck: boolean;
  logger?: SandboxPoolOptions["logger"];
}

interface AsyncSemaphoreWaiter {
  readonly ready: () => void;
  readonly reject: (reason?: unknown) => void;
  readonly signal?: AbortSignal;
}

interface AsyncSemaphoreLease {
  release?: () => void;
}

class AsyncSemaphore {
  private active = 0;
  private readonly waiters: AsyncSemaphoreWaiter[] = [];
  private readonly abortGroups = new Map<AbortSignal, {
    readonly waiters: Set<AsyncSemaphoreWaiter>;
    readonly onAbort: () => void;
  }>();

  constructor(private readonly limit: number) {}

  async acquire(signal?: AbortSignal): Promise<() => void> {
    signal?.throwIfAborted();
    if (this.active < this.limit) {
      this.active += 1;
      return () => this.release();
    }
    await new Promise<void>((resolve, reject) => {
      const waiter = {
        ready: () => {
          this.detachWaiter(waiter);
          this.active += 1;
          resolve();
        },
        reject,
        signal,
      };
      this.waiters.push(waiter);
      if (signal) {
        let group = this.abortGroups.get(signal);
        if (!group) {
          const waiters = new Set<AsyncSemaphoreWaiter>();
          const onAbort = () => {
            for (const abortedWaiter of waiters) {
              const index = this.waiters.indexOf(abortedWaiter);
              if (index >= 0) this.waiters.splice(index, 1);
              abortedWaiter.reject(signal.reason);
            }
            waiters.clear();
            this.abortGroups.delete(signal);
          };
          group = { waiters, onAbort };
          this.abortGroups.set(signal, group);
          signal.addEventListener("abort", onAbort, { once: true });
        }
        group.waiters.add(waiter);
      }
    });
    return () => this.release();
  }

  private detachWaiter(waiter: AsyncSemaphoreWaiter): void {
    const signal = waiter.signal;
    if (!signal) return;
    const group = this.abortGroups.get(signal);
    if (!group) return;
    group.waiters.delete(waiter);
    if (group.waiters.size === 0) {
      signal.removeEventListener("abort", group.onAbort);
      this.abortGroups.delete(signal);
    }
  }

  private release(): void {
    this.active -= 1;
    this.waiters.shift()?.ready();
  }
}

function makeOwnerId(): string {
  const random = globalThis.crypto?.randomUUID?.() ?? Math.random().toString(16).slice(2);
  return `pool-owner-${random}`;
}

function cloneConnectionConfig(config: ConnectionConfig): ConnectionConfig {
  return new ConnectionConfig({
    domain: config.domain,
    protocol: config.protocol,
    apiKey: config.apiKey,
    headers: { ...config.headers },
    requestTimeoutSeconds: config.requestTimeoutSeconds,
    debug: config.debug,
    useServerProxy: config.useServerProxy,
    endpointCacheTtlMs: config.endpointCacheTtlMs,
    endpointCacheSize: config.endpointCacheSize,
    endpointCacheDisabled: config.endpointCacheDisabled,
    disableMetrics: config.disableMetrics,
    enableTracing: config.enableTracing,
  });
}

function requireNonNegative(value: number, name: string): void {
  if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`);
}

function requirePositive(value: number, name: string): void {
  if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
}

function requireNonNegativeInteger(value: number, name: string): void {
  requireNonNegative(value, name);
  if (!Number.isInteger(value)) throw new Error(`${name} must be an integer`);
}

function policyRetries(policy: AcquirePolicy): boolean {
  return policy === AcquirePolicy.RETRY_NEXT_IDLE || policy === AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE;
}

function policyCreates(policy: AcquirePolicy): boolean {
  return policy === AcquirePolicy.DIRECT_CREATE || policy === AcquirePolicy.RETRY_NEXT_IDLE_THEN_CREATE;
}

function sleep(milliseconds: number, signal?: AbortSignal): Promise<void> {
  signal?.throwIfAborted();
  return new Promise((resolve, reject) => {
    const onDone = () => {
      signal?.removeEventListener("abort", onAbort);
      resolve();
    };
    const timer = setTimeout(onDone, milliseconds);
    if (!signal) return;
    const onAbort = () => {
      clearTimeout(timer);
      signal.removeEventListener("abort", onAbort);
      reject(signal.reason);
    };
    signal.addEventListener("abort", onAbort, { once: true });
  });
}

/** A client-side buffer of clean, ready sandboxes. */
export class SandboxPool {
  private readonly options: ResolvedPoolOptions;
  private manager?: SandboxManager;
  private lifecycleState = PoolLifecycleState.NOT_STARTED;
  private operatingState = PoolState.HEALTHY;
  private failureCount = 0;
  private lastError?: string;
  private inFlightOperations = 0;
  private timer?: ReturnType<typeof setInterval>;
  private heartbeatTimer?: ReturnType<typeof setInterval>;
  private startPromise?: Promise<void>;
  private reconcilePromise?: Promise<void>;
  private shutdownPromise?: Promise<void>;
  private abortController?: AbortController;
  private allowWarmupCommitWhileDraining = false;
  private runGeneration = 0;
  private leaderEpoch = 0;
  private primaryOwned = false;
  private readonly warmupTasks = new Map<Promise<void>, number>();
  private createSemaphore: AsyncSemaphore;
  private postCreateSemaphore: AsyncSemaphore;
  private readonly tracer: PoolTracer;
  private warmupCreateConfig?: ConnectionConfig;

  private constructor(options: SandboxPoolOptions) {
    const poolName = options.poolName?.trim();
    if (!poolName) throw new Error("poolName is required");
    if (options.ownerId !== undefined && !options.ownerId.trim()) {
      throw new Error("ownerId must not be blank");
    }
    requireNonNegativeInteger(options.maxIdle, "maxIdle");

    const idleTimeoutSeconds = options.idleTimeoutSeconds ?? DEFAULT_IDLE_TIMEOUT_SECONDS;
    const acquireMinRemainingTtlSeconds =
      options.acquireMinRemainingTtlSeconds ?? Math.min(idleTimeoutSeconds / 2, 60);
    const warmupConcurrency = options.warmupConcurrency ?? 128;
    const creationSpec = options.creationSpec ?? ({} as PoolCreationSpec);
    if (
      !options.sandboxCreator &&
      (creationSpec.image === undefined) === (creationSpec.snapshotId === undefined)
    ) {
      throw new Error("creationSpec must provide exactly one of image or snapshotId when sandboxCreator is absent");
    }

    requirePositive(idleTimeoutSeconds, "idleTimeoutSeconds");
    requirePositive(warmupConcurrency, "warmupConcurrency");
    if (!Number.isInteger(warmupConcurrency)) throw new Error("warmupConcurrency must be an integer");
    requirePositive(options.warmupCreateQps ?? 10, "warmupCreateQps");
    if (!Number.isInteger(options.warmupCreateQps ?? 10)) {
      throw new Error("warmupCreateQps must be an integer");
    }
    requirePositive(options.primaryLockTtlSeconds ?? 60, "primaryLockTtlSeconds");
    requirePositive(options.degradedThreshold ?? 3, "degradedThreshold");
    if (!Number.isInteger(options.degradedThreshold ?? 3)) {
      throw new Error("degradedThreshold must be an integer");
    }
    requireNonNegative(options.drainTimeoutSeconds ?? 30, "drainTimeoutSeconds");
    requireNonNegative(acquireMinRemainingTtlSeconds, "acquireMinRemainingTtlSeconds");
    if (acquireMinRemainingTtlSeconds >= idleTimeoutSeconds) {
      throw new Error("acquireMinRemainingTtlSeconds must be less than idleTimeoutSeconds");
    }
    requirePositive(options.acquireReadyTimeoutSeconds ?? DEFAULT_READY_TIMEOUT_SECONDS, "acquireReadyTimeoutSeconds");
    requirePositive(options.warmupReadyTimeoutSeconds ?? DEFAULT_READY_TIMEOUT_SECONDS, "warmupReadyTimeoutSeconds");
    requireNonNegative(options.warmupHealthCheckInitialDelayMillis ?? 0, "warmupHealthCheckInitialDelayMillis");
    requirePositive(
      options.acquireHealthCheckPollingIntervalMillis ?? DEFAULT_POLLING_INTERVAL_MILLIS,
      "acquireHealthCheckPollingIntervalMillis",
    );
    requirePositive(
      options.warmupHealthCheckPollingIntervalMillis ?? 500,
      "warmupHealthCheckPollingIntervalMillis",
    );
    requirePositive(options.maxAcquireRetries ?? 3, "maxAcquireRetries");
    if (!Number.isInteger(options.maxAcquireRetries ?? 3)) {
      throw new Error("maxAcquireRetries must be an integer");
    }
    requirePositive(
      options.warmupPostPrepareHealthCheckTimeoutSeconds ?? DEFAULT_READY_TIMEOUT_SECONDS,
      "warmupPostPrepareHealthCheckTimeoutSeconds",
    );

    this.options = {
      poolName,
      maxIdle: options.maxIdle,
      connectionConfig:
        options.connectionConfig instanceof ConnectionConfig
          ? cloneConnectionConfig(options.connectionConfig)
          : new ConnectionConfig(options.connectionConfig),
      creationSpec,
      sandboxCreator: options.sandboxCreator,
      stateStore: options.stateStore ?? new InMemoryPoolStateStore(),
      ownerId: options.ownerId?.trim() ?? makeOwnerId(),
      warmupCreateQps: options.warmupCreateQps ?? 10,
      warmupConcurrency,
      primaryLockTtlSeconds: options.primaryLockTtlSeconds ?? 60,
      degradedThreshold: options.degradedThreshold ?? 3,
      idleTimeoutSeconds,
      drainTimeoutSeconds: options.drainTimeoutSeconds ?? 30,
      acquireMinRemainingTtlSeconds,
      acquireReadyTimeoutSeconds: options.acquireReadyTimeoutSeconds ?? DEFAULT_READY_TIMEOUT_SECONDS,
      acquireHealthCheckPollingIntervalMillis:
        options.acquireHealthCheckPollingIntervalMillis ?? DEFAULT_POLLING_INTERVAL_MILLIS,
      acquireHealthCheck: options.acquireHealthCheck,
      acquireSkipHealthCheck: options.acquireSkipHealthCheck ?? false,
      maxAcquireRetries: options.maxAcquireRetries ?? 3,
      warmupReadyTimeoutSeconds: options.warmupReadyTimeoutSeconds ?? DEFAULT_READY_TIMEOUT_SECONDS,
      warmupHealthCheckInitialDelayMillis: options.warmupHealthCheckInitialDelayMillis ?? 0,
      warmupHealthCheckPollingIntervalMillis:
        options.warmupHealthCheckPollingIntervalMillis ?? 500,
      warmupHealthCheck: options.warmupHealthCheck,
      warmupSandboxPreparer: options.warmupSandboxPreparer,
      warmupPostPrepareHealthCheck: options.warmupPostPrepareHealthCheck,
      warmupPostPrepareHealthCheckTimeoutSeconds:
        options.warmupPostPrepareHealthCheckTimeoutSeconds ?? DEFAULT_READY_TIMEOUT_SECONDS,
      warmupSkipHealthCheck: options.warmupSkipHealthCheck ?? false,
      logger: options.logger,
    };
    this.createSemaphore = new AsyncSemaphore(Math.ceil(this.options.warmupCreateQps * CREATE_CONCURRENCY_HEADROOM));
    this.postCreateSemaphore = new AsyncSemaphore(warmupConcurrency);
    this.tracer = PoolTracer.from(this.options.connectionConfig);
  }

  static create(options: SandboxPoolOptions): SandboxPool {
    return new SandboxPool(options);
  }

  static inMemoryStateStore(): InMemoryPoolStateStore {
    return new InMemoryPoolStateStore();
  }

  async start(): Promise<void> {
    if (this.lifecycleState === PoolLifecycleState.RUNNING) return;
    if (this.lifecycleState === PoolLifecycleState.STARTING) {
      await this.startPromise;
      return;
    }
    if (this.lifecycleState === PoolLifecycleState.DRAINING) {
      throw new PoolNotRunningException(this.options.poolName, this.lifecycleState);
    }
    this.lifecycleState = PoolLifecycleState.STARTING;
    this.startPromise = (async () => {
      await this.reconcilePromise?.catch(() => undefined);
      await this.finishStart();
    })();
    try {
      await this.startPromise;
    } finally {
      this.startPromise = undefined;
    }
  }

  private async finishStart(): Promise<void> {
    try {
      await this.ensurePoolNamespaceActive();
      await this.options.stateStore.setMaxIdle(this.options.poolName, this.options.maxIdle);
      await this.options.stateStore.setIdleEntryTtl(this.options.poolName, this.options.idleTimeoutSeconds);
    } catch (cause) {
      if (this.lifecycleState === PoolLifecycleState.STARTING) {
        this.lifecycleState = PoolLifecycleState.STOPPED;
      }
      if (cause instanceof PoolDestroyedException || cause instanceof PoolStateStoreUnavailableException) throw cause;
      throw new PoolStateStoreUnavailableException("start", cause);
    }
    if (this.lifecycleState !== PoolLifecycleState.STARTING) {
      throw new PoolNotRunningException(this.options.poolName, this.lifecycleState);
    }

    this.abortController = new AbortController();
    this.runGeneration += 1;
    this.createSemaphore = new AsyncSemaphore(
      Math.ceil(this.options.warmupCreateQps * CREATE_CONCURRENCY_HEADROOM),
    );
    this.postCreateSemaphore = new AsyncSemaphore(this.options.warmupConcurrency);
    try {
      this.manager ??= this.createManager();
    } catch (cause) {
      if (this.lifecycleState === PoolLifecycleState.STARTING) {
        this.lifecycleState = PoolLifecycleState.STOPPED;
      }
      throw cause;
    }
    this.recordSuccess();
    this.lifecycleState = PoolLifecycleState.RUNNING;
    this.timer = setInterval(() => this.triggerReconcile(), 1_000);
    (this.timer as unknown as { unref?: () => void }).unref?.();
    const heartbeatIntervalMillis = Math.max(
      1,
      Math.min(1_000, Math.floor((this.options.primaryLockTtlSeconds * 1_000) / 3)),
    );
    const heartbeatGeneration = this.runGeneration;
    this.heartbeatTimer = setInterval(
      () => void this.runPrimaryHeartbeat(heartbeatGeneration),
      heartbeatIntervalMillis,
    );
    (this.heartbeatTimer as unknown as { unref?: () => void }).unref?.();
    if (this.options.primaryLockTtlSeconds <= this.options.warmupReadyTimeoutSeconds) {
      this.options.logger?.warn?.("pool primary lock TTL may expire during warmup", {
        poolName: this.options.poolName,
        primaryLockTtlSeconds: this.options.primaryLockTtlSeconds,
        warmupReadyTimeoutSeconds: this.options.warmupReadyTimeoutSeconds,
      });
    }
    if (this.options.maxIdle > 0) this.triggerReconcile();
  }

  async acquire(options: SandboxAcquireOptions = {}): Promise<Sandbox> {
    if (this.lifecycleState !== PoolLifecycleState.RUNNING) {
      await this.throwIfPoolNamespaceDestroyed();
      throw new PoolNotRunningException(this.options.poolName, this.lifecycleState);
    }
    options.signal?.throwIfAborted();
    const policy = options.policy ?? AcquirePolicy.DIRECT_CREATE;
    await this.ensurePoolNamespaceActiveForAcquire(policy);
    const generation = this.runGeneration;
    const maxAttempts = policyRetries(policy) ? this.options.maxAcquireRetries : 1;
    const minTtl = options.minRemainingTtlSeconds ?? this.options.acquireMinRemainingTtlSeconds;
    requireNonNegative(minTtl, "minRemainingTtlSeconds");
    if (options.sandboxTimeoutSeconds !== undefined) {
      requirePositive(options.sandboxTimeoutSeconds, "sandboxTimeoutSeconds");
    }

    this.inFlightOperations += 1;
    try {
      let attempted = false;
      let lastError: unknown;
      for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
        let result: TakeIdleResult;
        try {
          result = await this.options.stateStore.tryTakeIdleWithMinTtl(this.options.poolName, minTtl);
        } catch (cause) {
          if (cause instanceof PoolDestroyedException) throw cause;
          if (!policyCreates(policy)) throw new PoolStateStoreUnavailableException("tryTakeIdle", cause);
          this.options.logger?.warn?.("pool state store unavailable; falling back to direct create", {
            poolName: this.options.poolName,
            error: cause,
          });
          lastError = cause;
          break;
        }
        void this.killSandboxIds(result.discardedAliveSandboxIds).catch(() => undefined);
        if (!result.sandboxId) break;
        attempted = true;
        let sandbox: Sandbox | undefined;
        try {
          sandbox = await Sandbox.connect({
            sandboxId: result.sandboxId,
            connectionConfig: this.options.connectionConfig,
            adapterFactory: this.options.creationSpec.adapterFactory,
            skipHealthCheck: true,
            signal: options.signal,
          });
          if (!(options.skipHealthCheck ?? this.options.acquireSkipHealthCheck)) {
            await this.waitUntilHealthy(
              sandbox,
              this.options.acquireHealthCheck,
              this.options.acquireReadyTimeoutSeconds,
              this.options.acquireHealthCheckPollingIntervalMillis,
              options.signal,
            );
          }
        } catch (cause) {
          lastError = cause;
          await sandbox?.close().catch(() => undefined);
          void this.killSandboxId(result.sandboxId).catch(() => undefined);
          if (options.signal?.aborted) throw options.signal.reason;
          this.options.logger?.warn?.("pool idle sandbox connect or health check failed", {
            poolName: this.options.poolName,
            sandboxId: result.sandboxId,
            attempt: attempt + 1,
            error: cause,
          });
          await this.ensureAcquireStillRunning(generation);
          await this.ensurePoolNamespaceActive();
          if (!policyRetries(policy)) break;
          continue;
        }
        try {
          await this.renewAcquired(sandbox, options.sandboxTimeoutSeconds);
          await this.ensureAcquireStillRunning(generation);
        } catch (cause) {
          await this.killAndClose(sandbox);
          throw cause;
        }
        return sandbox;
      }

      await this.ensureAcquireStillRunning(generation);
      if (!policyCreates(policy)) {
        if (attempted) throw new PoolAcquireFailedException(this.options.poolName, lastError);
        throw new PoolEmptyException(this.options.poolName);
      }

      await this.ensurePoolNamespaceActiveForAcquire(policy);
      const sandbox = await this.createSandbox(PooledSandboxCreateReason.DIRECT_CREATE, options.signal);
      try {
        if (!(options.skipHealthCheck ?? this.options.acquireSkipHealthCheck)) {
          await this.waitUntilHealthy(
            sandbox,
            this.options.acquireHealthCheck,
            this.options.acquireReadyTimeoutSeconds,
            this.options.acquireHealthCheckPollingIntervalMillis,
            options.signal,
          );
        }
        await this.renewAcquired(sandbox, options.sandboxTimeoutSeconds);
        await this.ensurePoolNamespaceActiveForAcquire(policy);
        await this.ensureAcquireStillRunning(generation);
        return sandbox;
      } catch (cause) {
        await this.killAndClose(sandbox);
        throw cause;
      }
    } finally {
      this.inFlightOperations -= 1;
    }
  }

  async resize(maxIdle: number): Promise<void> {
    requireNonNegativeInteger(maxIdle, "maxIdle");
    await this.ensurePoolNamespaceActive();
    this.inFlightOperations += 1;
    try {
      await this.storeCall("setMaxIdle", () => this.options.stateStore.setMaxIdle(this.options.poolName, maxIdle));
      this.options.maxIdle = maxIdle;
    } finally {
      this.inFlightOperations -= 1;
    }
  }

  async releaseAllIdle(concurrency = 1): Promise<number> {
    requirePositive(concurrency, "concurrency");
    if (!Number.isInteger(concurrency)) throw new Error("concurrency must be an integer");
    this.inFlightOperations += 1;
    try {
      const sandboxIds: string[] = [];
      let drainFailure: unknown;
      while (true) {
        try {
          const sandboxId = await this.storeCall("tryTakeIdle", () =>
            this.options.stateStore.tryTakeIdle(this.options.poolName),
          );
          if (!sandboxId) break;
          sandboxIds.push(sandboxId);
        } catch (error) {
          drainFailure = error;
          break;
        }
      }
      let next = 0;
      await Promise.all(
        Array.from({ length: Math.min(concurrency, sandboxIds.length) }, async () => {
          while (next < sandboxIds.length) {
            const sandboxId = sandboxIds[next++];
            await this.killSandboxId(sandboxId!);
          }
        }),
      );
      if (drainFailure !== undefined) throw drainFailure;
      return sandboxIds.length;
    } finally {
      this.inFlightOperations -= 1;
    }
  }

  async snapshot(): Promise<PoolSnapshot> {
    const counters = await this.storeCall("snapshotCounters", () =>
      this.options.stateStore.snapshotCounters(this.options.poolName),
    );
    return {
      lifecycleState: this.lifecycleState,
      state:
        this.lifecycleState === PoolLifecycleState.DRAINING
          ? PoolState.DRAINING
          : this.lifecycleState === PoolLifecycleState.NOT_STARTED || this.lifecycleState === PoolLifecycleState.STOPPED
            ? PoolState.STOPPED
            : this.operatingState,
      idleCount: counters.idleCount,
      maxIdle:
        (await this.storeCall("getMaxIdle", () =>
          this.options.stateStore.getMaxIdle(this.options.poolName),
        )) ?? this.options.maxIdle,
      failureCount: this.failureCount,
      backoffActive: false,
      lastError: this.lastError,
      inFlightOperations: this.inFlightOperations,
    };
  }

  snapshotIdleEntries(): Promise<IdleEntry[]> {
    return this.storeCall("snapshotIdleEntries", () =>
      this.options.stateStore.snapshotIdleEntries(this.options.poolName),
    );
  }

  async shutdown(graceful = true): Promise<void> {
    if (this.lifecycleState === PoolLifecycleState.NOT_STARTED || this.lifecycleState === PoolLifecycleState.STOPPED) {
      this.lifecycleState = PoolLifecycleState.STOPPED;
      return;
    }
    if (this.lifecycleState === PoolLifecycleState.DRAINING) {
      await this.shutdownPromise;
      return;
    }
    this.lifecycleState = PoolLifecycleState.DRAINING;
    this.allowWarmupCommitWhileDraining = graceful;
    this.shutdownPromise = this.finishShutdown(graceful);
    await this.shutdownPromise;
  }

  private async finishShutdown(graceful: boolean): Promise<void> {
    try {
      await this.startPromise?.catch(() => undefined);
      if (this.timer) clearInterval(this.timer);
      this.timer = undefined;
      if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
      this.heartbeatTimer = undefined;

      if (graceful) {
        const deadline = Date.now() + this.options.drainTimeoutSeconds * 1000;
        while ((this.inFlightOperations > 0 || this.reconcilePromise) && Date.now() < deadline) {
          await sleep(10);
        }
      }
      this.allowWarmupCommitWhileDraining = false;
      this.abortController?.abort(new Error("Sandbox pool is shutting down"));
      try {
        await this.options.stateStore.releasePrimaryLock(this.options.poolName, this.options.ownerId);
      } catch {
        // Best-effort release; a TTL-backed lock will expire naturally.
      }
      try {
        await this.manager?.close();
      } catch {
        // Shutdown still transitions to STOPPED when transport cleanup fails.
      }
      this.manager = undefined;
      this.markPrimaryLost();
    } finally {
      this.allowWarmupCommitWhileDraining = false;
      this.lifecycleState = PoolLifecycleState.STOPPED;
      this.shutdownPromise = undefined;
    }
  }

  close(): Promise<void> {
    return this.shutdown(true);
  }

  private triggerReconcile(): void {
    if (this.lifecycleState !== PoolLifecycleState.RUNNING || this.reconcilePromise) return;
    this.reconcilePromise = this.reconcile()
      .catch((error: unknown) => {
        this.markPrimaryLost();
        this.recordFailure(error);
      })
      .finally(() => {
        this.reconcilePromise = undefined;
      });
  }

  private async reconcile(): Promise<void> {
    const { poolName, ownerId, primaryLockTtlSeconds, stateStore } = this.options;
    const destroyState = await stateStore.getDestroyState(poolName);
    if (destroyState !== PoolDestroyState.ACTIVE) {
      await this.stopAfterNamespaceDestroyed(destroyState);
      return;
    }
    if (!(await stateStore.tryAcquirePrimaryLock(poolName, ownerId, primaryLockTtlSeconds))) {
      this.markPrimaryLost();
      return;
    }
    this.markPrimaryAcquired();

    const reaped = await stateStore.reapExpiredIdleWithMinTtl(
      poolName,
      new Date(),
      this.options.acquireMinRemainingTtlSeconds,
    );
    await this.killSandboxIds(reaped.discardedAliveSandboxIds);

    const maxIdle = (await stateStore.getMaxIdle(poolName)) ?? this.options.maxIdle;
    let idleCount = (await stateStore.snapshotCounters(poolName)).idleCount;
    while (idleCount > maxIdle) {
      const sandboxId = await stateStore.tryTakeIdle(poolName);
      if (!sandboxId) break;
      await this.killSandboxId(sandboxId);
      idleCount -= 1;
    }

    const warmingCount = [...this.warmupTasks.values()].filter((value) => value === this.runGeneration).length;
    const deficit = Math.max(0, maxIdle - idleCount - warmingCount);
    const createCount = Math.min(deficit, this.options.warmupCreateQps);
    const generation = this.runGeneration;
    const leaderEpoch = this.leaderEpoch;
    const signal = this.abortController?.signal;
    const postCreateSemaphore = this.postCreateSemaphore;
    for (let index = 0; index < createCount; index += 1) {
      const taskAbortController = new AbortController();
      const unsubscribeAbort = subscribeAbort(
        signal,
        () => taskAbortController.abort(signal?.reason),
      );
      const task = this.tracer.runWarmup(
        {
          "pool.name": poolName,
          "pool.owner": ownerId,
          "pool.run.generation": generation,
          "pool.leader.epoch": this.leaderEpoch,
        },
        () => this.createIdleSandbox(
          generation,
          leaderEpoch,
          postCreateSemaphore,
          taskAbortController.signal,
        ),
      )
        .then(() => this.recordSuccess())
        .catch((error: unknown) => {
          this.recordFailure(error);
          this.options.logger?.warn?.("pool warmup sandbox creation failed", { poolName, error });
        })
        .finally(() => {
          unsubscribeAbort();
          this.warmupTasks.delete(task);
        });
      this.warmupTasks.set(task, generation);
    }
  }

  private async createIdleSandbox(
    generation: number,
    leaderEpoch: number,
    postCreateSemaphore: AsyncSemaphore,
    signal?: AbortSignal,
  ): Promise<void> {
    this.inFlightOperations += 1;
    let sandbox: Sandbox | undefined;
    let committed = false;
    try {
      const releaseCreate = await this.createSemaphore.acquire(signal);
      try {
        sandbox = await this.tracer.runPhase(POOL_WARMUP_SPANS.create, async () =>
          await this.createSandbox(PooledSandboxCreateReason.WARMUP, signal, true),
        );
      } finally {
        releaseCreate();
      }
      const warmupReadinessDeadline = performance.now() + this.options.warmupReadyTimeoutSeconds * 1_000;
      // Decide the final attempt before sleeping, as Kotlin does: the delay timer can fire slightly
      // before performance.now() reaches the deadline. Compare in seconds, because 2.007 * 1_000 is
      // 2007.0000000000002 and a millisecond comparison would miss equal settings.
      const warmupFinalAttemptOnly =
        this.options.warmupHealthCheckInitialDelayMillis / 1_000 >= this.options.warmupReadyTimeoutSeconds;
      if (!this.options.warmupSkipHealthCheck && this.options.warmupHealthCheckInitialDelayMillis > 0) {
        await sleep(
          Math.min(
            this.options.warmupHealthCheckInitialDelayMillis,
            this.options.warmupReadyTimeoutSeconds * 1_000,
          ),
          signal,
        );
      }
      const lease: AsyncSemaphoreLease = {};
      try {
        if (!this.options.warmupSkipHealthCheck) {
          await this.tracer.runPhase(POOL_WARMUP_SPANS.readiness, async () =>
            await this.waitUntilWarmupHealthy(
              sandbox!,
              warmupReadinessDeadline,
              warmupFinalAttemptOnly,
              postCreateSemaphore,
              lease,
              signal,
            ),
          );
        } else {
          lease.release = await postCreateSemaphore.acquire(signal);
        }
        if (this.options.warmupSandboxPreparer) {
          await this.tracer.runPhase(POOL_WARMUP_SPANS.prepare, async () =>
            await runAbortable(
              () => this.options.warmupSandboxPreparer!(sandbox!),
              signal,
            ),
          );
        }
        if (this.options.warmupPostPrepareHealthCheck) {
          await this.tracer.runPhase(POOL_WARMUP_SPANS.postPrepareReadiness, async () =>
            await this.waitUntilHealthyWithSemaphore(
              sandbox!,
              this.options.warmupPostPrepareHealthCheck!,
              this.options.warmupPostPrepareHealthCheckTimeoutSeconds,
              this.options.warmupHealthCheckPollingIntervalMillis,
              postCreateSemaphore,
              lease,
              signal,
            ),
          );
        }
        await this.tracer.runPhase(POOL_WARMUP_SPANS.renew, async () =>
          await sandbox!.renew(this.options.idleTimeoutSeconds),
        );
        const didCommit = await this.tracer.runPhase(POOL_WARMUP_SPANS.commit, async () => {
          const stillPrimary = await this.options.stateStore.renewPrimaryLock(
            this.options.poolName,
            this.options.ownerId,
            this.options.primaryLockTtlSeconds,
          );
          const mayCommit =
            generation === this.runGeneration &&
            leaderEpoch === this.leaderEpoch &&
            this.primaryOwned &&
            (this.lifecycleState === PoolLifecycleState.RUNNING ||
              (this.lifecycleState === PoolLifecycleState.DRAINING && this.allowWarmupCommitWhileDraining));
          if (!stillPrimary || !mayCommit) return false;
          await this.options.stateStore.putIdle(this.options.poolName, String(sandbox!.id));
          return true;
        });
        if (!didCommit) {
          await this.killAndClose(sandbox);
          return;
        }
        committed = true;
        await sandbox.close().catch((error: unknown) => {
          this.options.logger?.warn?.("failed to close pooled sandbox client", {
            sandboxId: String(sandbox!.id),
            error,
          });
        });
      } finally {
        lease.release?.();
        lease.release = undefined;
      }
    } catch (cause) {
      if (sandbox && !committed) await this.killAndClose(sandbox);
      throw cause;
    } finally {
      this.inFlightOperations -= 1;
    }
  }

  private async createSandbox(
    reason: PooledSandboxCreateReason,
    signal?: AbortSignal,
    injectTraceContext = false,
  ): Promise<Sandbox> {
    const baseCreateConfig =
      reason === PooledSandboxCreateReason.WARMUP
        ? (this.warmupCreateConfig ??= this.options.connectionConfig.withCreateMetricsDisabled())
        : this.options.connectionConfig;
    const createConnectionConfig = injectTraceContext
      ? this.tracer.createConnectionConfig(baseCreateConfig)
      : baseCreateConfig;
    if (this.options.sandboxCreator) {
      return await this.options.sandboxCreator({
        poolName: this.options.poolName,
        ownerId: this.options.ownerId,
        idleTimeoutSeconds: this.options.idleTimeoutSeconds,
        reason,
        readyTimeoutSeconds:
          reason === PooledSandboxCreateReason.WARMUP
            ? this.options.warmupReadyTimeoutSeconds
            : this.options.acquireReadyTimeoutSeconds,
        healthCheckPollingIntervalMillis:
          reason === PooledSandboxCreateReason.WARMUP
            ? this.options.warmupHealthCheckPollingIntervalMillis
            : this.options.acquireHealthCheckPollingIntervalMillis,
        skipHealthCheck: true,
        healthCheck:
          reason === PooledSandboxCreateReason.WARMUP
            ? this.options.warmupHealthCheck
            : this.options.acquireHealthCheck,
        connectionConfig: this.options.connectionConfig,
        createConnectionConfig,
        creationSpec: this.options.creationSpec,
        signal,
      });
    }
    return await Sandbox.create({
      ...this.options.creationSpec,
      connectionConfig: createConnectionConfig,
      timeoutSeconds: this.options.idleTimeoutSeconds,
      skipHealthCheck: true,
      signal,
    });
  }

  private async waitUntilHealthy(
    sandbox: Sandbox,
    healthCheck: PoolHealthCheck | undefined,
    timeoutSeconds: number,
    pollingIntervalMillis: number,
    signal?: AbortSignal,
  ): Promise<void> {
    if (typeof sandbox.waitUntilReady === "function") {
      await sandbox.waitUntilReady({
        readyTimeoutSeconds: timeoutSeconds,
        pollingIntervalMillis,
        healthCheck,
        signal,
      });
      return;
    }

    // Custom creators may return compatible objects with only isHealthy().
    const budget = new ReadinessBudget(timeoutSeconds, signal);
    budget.healthContext(`domain=${this.options.connectionConfig.domain}, useServerProxy=${this.options.connectionConfig.useServerProxy}`);
    while (true) {
      try {
        budget.attempt();
        const healthy = await budget.run(async () => healthCheck ? await healthCheck(sandbox) : await sandbox.isHealthy());
        if (healthy) return;
        budget.record("Health check returned false continuously.");
      } catch (error) {
        budget.remaining();
        budget.record(error);
      }
      await budget.pause(pollingIntervalMillis);
    }
  }

  private async waitUntilHealthyWithSemaphore(
    sandbox: Sandbox,
    healthCheck: PoolHealthCheck,
    timeoutSeconds: number,
    pollingIntervalMillis: number,
    semaphore: AsyncSemaphore,
    lease: AsyncSemaphoreLease,
    signal?: AbortSignal,
  ): Promise<void> {
    const budget = new ReadinessBudget(timeoutSeconds, signal);
    budget.healthContext(`domain=${this.options.connectionConfig.domain}, useServerProxy=${this.options.connectionConfig.useServerProxy}`);
    while (true) {
      let healthy = false;
      try {
        lease.release ??= await semaphore.acquire(signal);
        budget.attempt();
        healthy = await budget.run(async () => await healthCheck(sandbox));
        if (healthy) return;
        budget.record("Health check returned false continuously.");
      } catch (error) {
        budget.remaining();
        budget.record(error);
      } finally {
        if (!healthy) {
          lease.release?.();
          lease.release = undefined;
        }
      }
      await budget.pause(pollingIntervalMillis);
    }
  }

  private async waitUntilWarmupHealthy(
    sandbox: Sandbox,
    deadline: number,
    finalAttemptOnly: boolean,
    semaphore: AsyncSemaphore,
    lease: AsyncSemaphoreLease,
    signal?: AbortSignal,
  ): Promise<void> {
    const remainingMillis = deadline - performance.now();
    if (!finalAttemptOnly && remainingMillis > 0) {
      await this.waitUntilHealthyWithSemaphore(
        sandbox,
        this.options.warmupHealthCheck ?? ((current) => current.isHealthy()),
        remainingMillis / 1_000,
        this.options.warmupHealthCheckPollingIntervalMillis,
        semaphore,
        lease,
        signal,
      );
      return;
    }

    signal?.throwIfAborted();
    lease.release ??= await semaphore.acquire(signal);
    const healthy = await runAbortable(() => this.options.warmupHealthCheck
      ? this.options.warmupHealthCheck(sandbox)
      : sandbox.isHealthy(), signal);
    if (!healthy) {
      throw new SandboxReadyTimeoutException({
        message: `Sandbox warmup readiness timed out after ${this.options.warmupReadyTimeoutSeconds}s`,
      });
    }
  }

  private async renewAcquired(sandbox: Sandbox, timeoutSeconds?: number): Promise<void> {
    if (timeoutSeconds === undefined) return;
    await sandbox.renew(timeoutSeconds);
  }

  private async killSandboxIds(sandboxIds: string[]): Promise<void> {
    if (sandboxIds.length === 0) return;
    this.inFlightOperations += 1;
    try {
      const pooledManager = this.manager;
      let manager: SandboxManager;
      try {
        manager = pooledManager ?? this.createManager();
      } catch (error) {
        this.options.logger?.warn?.("failed to create manager for pooled sandbox cleanup", { error });
        return;
      }
      try {
        for (const sandboxId of sandboxIds) {
          try {
            await manager.killSandbox(sandboxId);
          } catch (error) {
            this.options.logger?.warn?.("failed to kill pooled sandbox", { sandboxId, error });
          }
        }
      } finally {
        if (!pooledManager) await manager.close().catch(() => undefined);
      }
    } finally {
      this.inFlightOperations -= 1;
    }
  }

  private async killSandboxId(sandboxId: string): Promise<void> {
    await this.killSandboxIds([sandboxId]);
  }

  private async killAndClose(sandbox: Sandbox): Promise<void> {
    await sandbox.kill().catch(() => undefined);
    await sandbox.close().catch(() => undefined);
  }

  private async ensureAcquireStillRunning(generation: number): Promise<void> {
    if (this.lifecycleState === PoolLifecycleState.RUNNING && generation === this.runGeneration) return;
    throw new PoolNotRunningException(this.options.poolName, this.lifecycleState);
  }

  private async ensurePoolNamespaceActive(): Promise<void> {
    const state = await this.storeCall("getDestroyState", () =>
      this.options.stateStore.getDestroyState(this.options.poolName),
    );
    if (state !== PoolDestroyState.ACTIVE) {
      throw new PoolDestroyedException(this.options.poolName, state);
    }
  }

  private async throwIfPoolNamespaceDestroyed(): Promise<void> {
    try {
      await this.ensurePoolNamespaceActive();
    } catch (error) {
      if (error instanceof PoolDestroyedException) throw error;
      // A stopped local pool remains observably stopped when the shared store
      // cannot answer the stronger destroy-state question.
    }
  }

  private async ensurePoolNamespaceActiveForAcquire(policy: AcquirePolicy): Promise<void> {
    try {
      await this.ensurePoolNamespaceActive();
    } catch (error) {
      if (error instanceof PoolDestroyedException) throw error;
      if (!policyCreates(policy)) throw error;
      this.options.logger?.warn?.(
        "pool state store unavailable during destroy-state check; falling back to direct create",
        { poolName: this.options.poolName, policy, error },
      );
    }
  }

  private async runPrimaryHeartbeat(generation: number): Promise<void> {
    if (
      generation !== this.runGeneration ||
      !this.primaryOwned ||
      (this.lifecycleState !== PoolLifecycleState.RUNNING &&
        this.lifecycleState !== PoolLifecycleState.DRAINING)
    ) return;
    try {
      const renewed = await this.options.stateStore.renewPrimaryLock(
        this.options.poolName,
        this.options.ownerId,
        this.options.primaryLockTtlSeconds,
      );
      if (generation !== this.runGeneration) return;
      if (!renewed) this.markPrimaryLost();
    } catch (error) {
      this.options.logger?.warn?.("pool primary heartbeat failed", {
        poolName: this.options.poolName,
        error,
      });
    }
  }

  private async stopAfterNamespaceDestroyed(state: PoolDestroyState): Promise<void> {
    if (this.lifecycleState !== PoolLifecycleState.RUNNING) return;
    this.lifecycleState = PoolLifecycleState.STOPPED;
    if (this.timer) clearInterval(this.timer);
    this.timer = undefined;
    if (this.heartbeatTimer) clearInterval(this.heartbeatTimer);
    this.heartbeatTimer = undefined;
    this.abortController?.abort(new PoolDestroyedException(this.options.poolName, state));
    try {
      await this.options.stateStore.releasePrimaryLock(this.options.poolName, this.options.ownerId);
    } catch {
      // Destroy fencing already prevents future commits; lock release is best effort.
    }
    this.markPrimaryLost();
    await this.manager?.close().catch(() => undefined);
    this.manager = undefined;
  }

  private markPrimaryAcquired(): void {
    if (this.primaryOwned) return;
    this.primaryOwned = true;
    this.leaderEpoch += 1;
  }

  private markPrimaryLost(): void {
    if (!this.primaryOwned) return;
    this.primaryOwned = false;
    this.leaderEpoch += 1;
  }

  private recordSuccess(): void {
    this.failureCount = 0;
    this.lastError = undefined;
    this.operatingState = PoolState.HEALTHY;
  }

  private recordFailure(error: unknown): void {
    this.failureCount += 1;
    this.lastError = error instanceof Error ? error.message : String(error);
    if (this.failureCount >= this.options.degradedThreshold) {
      this.operatingState = PoolState.DEGRADED;
    }
    this.options.logger?.warn?.("pool reconcile failed", {
      poolName: this.options.poolName,
      error,
      failureCount: this.failureCount,
    });
  }

  private createManager(): SandboxManager {
    return SandboxManager.create({
      connectionConfig: this.options.connectionConfig,
      adapterFactory: this.options.creationSpec.adapterFactory,
    });
  }

  private async storeCall<T>(operation: string, call: () => Promise<T>): Promise<T> {
    try {
      return await call();
    } catch (cause) {
      if (cause instanceof PoolStateStoreUnavailableException || cause instanceof PoolDestroyedException) throw cause;
      throw new PoolStateStoreUnavailableException(operation, cause);
    }
  }
}

async function runAbortable<T>(action: () => T | Promise<T>, signal?: AbortSignal): Promise<T> {
  signal?.throwIfAborted();
  if (!signal) return await action();

  let rejectOnAbort: (() => void) | undefined;
  const aborted = new Promise<never>((_, reject) => {
    rejectOnAbort = () => reject(signal.reason);
    signal.addEventListener("abort", rejectOnAbort, { once: true });
  });
  try {
    return await Promise.race([Promise.resolve().then(action), aborted]);
  } finally {
    if (rejectOnAbort) signal.removeEventListener("abort", rejectOnAbort);
  }
}