Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
fix(sdks): raise typed, actionable errors when sandbox dies mid-request (#1419)
Mish Ushakov82add5b4ea25
diff --git a/.changeset/spotty-llamas-shout.md b/.changeset/spotty-llamas-shout.mdnew file mode 100644index 000000000..9249e70ef--- /dev/null+++ b/.changeset/spotty-llamas-shout.md@@ -0,0 +1,6 @@+---+'e2b': patch+'@e2b/python-sdk': patch+---++Raise a typed, actionable error when the sandbox dies while a request is in flight. When the connection is dropped mid-request (streaming RPC calls — commands, PTY, directory watch — and filesystem read/write), the SDKs now probe the sandbox health endpoint: if the sandbox is confirmed gone, a `TimeoutError` (JS) / `TimeoutException` (Python) is raised stating the sandbox was killed or reached its end of life — consistent with how requests to an already-dead sandbox surface. In all other cases the original error propagates unchanged.diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.tsindex 52529c981..9886cd0da 100644--- a/packages/js-sdk/src/envd/api.ts+++ b/packages/js-sdk/src/envd/api.ts@@ -12,10 +12,12 @@ import {formatSandboxTimeoutError,AuthenticationError,RateLimitError,+ TimeoutError,} from '../errors'import { StartResponse, ConnectResponse } from './process/process_pb'import { Code, ConnectError } from '@connectrpc/connect'import { WatchDirResponse } from './filesystem/filesystem_pb'+import { SandboxHealthCheck } from './rpc'type ApiError = { message?: string } | string@@ -29,6 +31,64 @@ const DEFAULT_ERROR_MAP: Record<number, (message: string) => Error> = {507: (message) => new NotEnoughSpaceError(message),}+const HEALTH_CHECK_TIMEOUT_MS = 5_000++/**+ * Probes the sandbox's envd health endpoint.+ *+ * @param envdApi - The envd API client of the sandbox.+ * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if its state could not be determined.+ */+export async function checkSandboxHealth(+ envdApi: EnvdApiClient+): Promise<boolean | undefined> {+ try {+ const res = await envdApi.api.GET('/health', {+ signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS),+ })++ if (res.response.status === 502) {+ return false+ }+ if (res.response.ok) {+ return true+ }++ return undefined+ } catch {+ return undefined+ }+}++/**+ * Handles transport-level fetch failures from envd API calls. When the connection was+ * dropped mid-request, probes the sandbox health to tell apart the sandbox being killed+ * from a transient network failure (e.g. a load balancer dropping the connection).+ *+ * @param err - The caught error, expected to be a fetch transport failure.+ * @param checkHealth - Probe returning whether the sandbox is running, or `undefined` when unknown.+ * @returns A `TimeoutError` when the connection was terminated mid-request and the sandbox is confirmed gone, or the original error otherwise.+ */+export async function handleEnvdApiFetchError(+ err: unknown,+ checkHealth?: SandboxHealthCheck+): Promise<Error> {+ // Undici surfaces a connection dropped mid-body as a TypeError with the message 'terminated'+ if (err instanceof TypeError && err.message === 'terminated') {+ const running = checkHealth+ ? await checkHealth().catch(() => undefined)+ : undefined++ if (running === false) {+ return new TimeoutError(+ `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.`+ )+ }+ }++ return err as Error+}+/*** Handles errors from envd API responses by mapping HTTP status codes to specific error types.*diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.tsindex 1b559c72e..1e9aa4a3b 100644--- a/packages/js-sdk/src/envd/rpc.ts+++ b/packages/js-sdk/src/envd/rpc.ts@@ -14,6 +14,25 @@ import {} from '../errors'import { ENVD_DEFAULT_USER } from './versions'+/**+ * Result of a sandbox health probe: `true` if the sandbox is running, `false` if it is not,+ * `undefined` if its state could not be determined.+ */+export type SandboxHealthCheck = () => Promise<boolean | undefined>++/**+ * Checks whether the error is the signature of the connection to the sandbox being+ * dropped mid-request — an HTTP/2 stream reset surfaced by connect as `Code.Unknown`+ * with the message 'terminated'.+ */+export function isConnectionTerminatedError(err: unknown): boolean {+ return (+ err instanceof ConnectError &&+ err.code === Code.Unknown &&+ err.rawMessage === 'terminated'+ )+}+const DEFAULT_ERROR_MAP: Partial<Record<Code, (message: string) => Error>> = {[Code.InvalidArgument]: (message) => new InvalidArgumentError(message),[Code.Unauthenticated]: (message) => new AuthenticationError(message),@@ -62,6 +81,36 @@ export function handleRpcError(return err as Error}+/**+ * Like {@link handleRpcError}, but when the connection to the sandbox was dropped+ * mid-request it probes the sandbox health to tell apart the sandbox being killed+ * from a transient network failure (e.g. a load balancer dropping the connection).+ * When the probe confirms the sandbox is gone, a `TimeoutError` is returned —+ * consistent with how requests to an already-dead sandbox surface.+ *+ * @param err - The caught error, expected to be a `ConnectError` from the gRPC transport.+ * @param checkHealth - Probe returning whether the sandbox is running, or `undefined` when unknown.+ * @param errorMap - Optional map of gRPC `Code` values to error factory functions that override the defaults.+ * @returns The corresponding `Error` instance.+ */+export async function handleRpcErrorWithHealthCheck(+ err: unknown,+ checkHealth?: SandboxHealthCheck,+ errorMap?: Partial<Record<Code, (message: string) => Error>>+): Promise<Error> {+ if (isConnectionTerminatedError(err) && checkHealth) {+ const running = await checkHealth().catch(() => undefined)++ if (running === false) {+ return new TimeoutError(+ `${(err as ConnectError).message}: The sandbox was killed or reached its end of life while the request was in flight.`+ )+ }+ }++ return handleRpcError(err, errorMap)+}+function encode64(value: string): string {switch (runtime) {case 'deno':diff --git a/packages/js-sdk/src/sandbox/commands/commandHandle.ts b/packages/js-sdk/src/sandbox/commands/commandHandle.tsindex 09713bbb8..0bf37bdb0 100644--- a/packages/js-sdk/src/sandbox/commands/commandHandle.ts+++ b/packages/js-sdk/src/sandbox/commands/commandHandle.ts@@ -1,4 +1,7 @@-import { handleRpcError } from '../../envd/rpc'+import {+ handleRpcErrorWithHealthCheck,+ SandboxHealthCheck,+} from '../../envd/rpc'import { SandboxError } from '../../errors'import { ConnectResponse, StartResponse } from '../../envd/process/process_pb'import type { CommandRequestOpts } from '.'@@ -112,7 +115,8 @@ export class CommandHandle) => Promise<void>,private readonly handleCloseStdin?: (opts?: CommandRequestOpts- ) => Promise<void>+ ) => Promise<void>,+ private readonly checkHealth?: SandboxHealthCheck) {this._wait = this.handleEvents()}@@ -281,7 +285,10 @@ export class CommandHandle}}} catch (e) {- this.iterationError = handleRpcError(e)+ this.iterationError = await handleRpcErrorWithHealthCheck(+ e,+ this.checkHealth+ )} finally {this.handleDisconnect()}diff --git a/packages/js-sdk/src/sandbox/commands/index.ts b/packages/js-sdk/src/sandbox/commands/index.tsindex 6302ed2d9..bbc92241f 100644--- a/packages/js-sdk/src/sandbox/commands/index.ts+++ b/packages/js-sdk/src/sandbox/commands/index.ts@@ -15,12 +15,20 @@ import {setupRequestController,Username,} from '../../connectionConfig'-import { handleProcessStartEvent } from '../../envd/api'+import {+ checkSandboxHealth,+ EnvdApiClient,+ handleProcessStartEvent,+} from '../../envd/api'import {Process as ProcessService,Signal,} from '../../envd/process/process_pb'-import { authenticationHeader, handleRpcError } from '../../envd/rpc'+import {+ authenticationHeader,+ handleRpcErrorWithHealthCheck,+ SandboxHealthCheck,+} from '../../envd/rpc'import { ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE } from '../../envd/versions'import { SandboxError } from '../../errors'import { CommandHandle, CommandResult } from './commandHandle'@@ -129,16 +137,16 @@ export class Commands {private readonly defaultProcessConnectionTimeout = 60_000 // 60 secondsprivate readonly envdVersion: string+ private readonly checkHealth: SandboxHealthCheckconstructor(transport: Transport,- private readonly connectionConfig: ConnectionConfig,- metadata: {- version: string- }+ private readonly envdApi: EnvdApiClient,+ private readonly connectionConfig: ConnectionConfig) {this.rpc = createClient(ProcessService, transport)- this.envdVersion = metadata.version+ this.envdVersion = envdApi.version+ this.checkHealth = () => checkSandboxHealth(this.envdApi)}/**@@ -177,7 +185,7 @@ export class Commands {...(p.config!.cwd && { cwd: p.config!.cwd }),}))} catch (err) {- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -220,7 +228,7 @@ export class Commands {})} catch (err) {- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -257,7 +265,7 @@ export class Commands {})} catch (err) {- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -298,7 +306,7 @@ export class Commands {}}- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -354,11 +362,12 @@ export class Commands {opts?.onStderr,undefined,(data, stdinOpts) => this.sendStdin(pid, data, stdinOpts),- (stdinOpts) => this.closeStdin(pid, stdinOpts)+ (stdinOpts) => this.closeStdin(pid, stdinOpts),+ this.checkHealth)} catch (err) {cleanup()- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -468,11 +477,12 @@ export class Commands {opts?.onStderr,undefined,(data, stdinOpts) => this.sendStdin(pid, data, stdinOpts),- (stdinOpts) => this.closeStdin(pid, stdinOpts)+ (stdinOpts) => this.closeStdin(pid, stdinOpts),+ this.checkHealth)} catch (err) {cleanup()- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}}diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.tsindex 089361250..283190bf0 100644--- a/packages/js-sdk/src/sandbox/commands/pty.ts+++ b/packages/js-sdk/src/sandbox/commands/pty.ts@@ -19,8 +19,16 @@ import {setupRequestController,} from '../../connectionConfig'import { CommandHandle } from './commandHandle'-import { authenticationHeader, handleRpcError } from '../../envd/rpc'-import { handleProcessStartEvent } from '../../envd/api'+import {+ authenticationHeader,+ handleRpcErrorWithHealthCheck,+ SandboxHealthCheck,+} from '../../envd/rpc'+import {+ checkSandboxHealth,+ EnvdApiClient,+ handleProcessStartEvent,+} from '../../envd/api'export interface PtyCreateOptsextends Pick<ConnectionOpts, 'requestTimeoutMs' | 'signal'> {@@ -74,18 +82,18 @@ export type PtyConnectOpts = Pick<PtyCreateOpts, 'onData' | 'timeoutMs'> &export class Pty {private readonly rpc: Client<typeof ProcessService>private readonly envdVersion: string+ private readonly checkHealth: SandboxHealthCheckprivate readonly defaultPtyConnectionTimeout = 60_000 // 60 secondsconstructor(private readonly transport: Transport,- private readonly connectionConfig: ConnectionConfig,- metadata: {- version: string- }+ private readonly envdApi: EnvdApiClient,+ private readonly connectionConfig: ConnectionConfig) {this.rpc = createClient(ProcessService, this.transport)- this.envdVersion = metadata.version+ this.envdVersion = envdApi.version+ this.checkHealth = () => checkSandboxHealth(this.envdApi)}/**@@ -144,11 +152,14 @@ export class Pty {events,undefined,undefined,- opts.onData+ opts.onData,+ undefined,+ undefined,+ this.checkHealth)} catch (err) {cleanup()- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -198,11 +209,14 @@ export class Pty {events,undefined,undefined,- opts?.onData+ opts?.onData,+ undefined,+ undefined,+ this.checkHealth)} catch (err) {cleanup()- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -242,7 +256,7 @@ export class Pty {})} catch (err) {- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -283,7 +297,7 @@ export class Pty {})} catch (err) {- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}@@ -327,7 +341,7 @@ export class Pty {}}- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}}diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.tsindex e63e9ce3b..116361ff0 100644--- a/packages/js-sdk/src/sandbox/filesystem/index.ts+++ b/packages/js-sdk/src/sandbox/filesystem/index.ts@@ -15,8 +15,17 @@ import {Username,} from '../../connectionConfig'-import { handleEnvdApiError, handleWatchDirStartEvent } from '../../envd/api'-import { authenticationHeader, handleRpcError } from '../../envd/rpc'+import {+ checkSandboxHealth,+ handleEnvdApiError,+ handleEnvdApiFetchError,+ handleWatchDirStartEvent,+} from '../../envd/api'+import {+ authenticationHeader,+ handleRpcErrorWithHealthCheck,+ SandboxHealthCheck,+} from '../../envd/rpc'import { EnvdApiClient } from '../../envd/api'import {@@ -54,8 +63,15 @@ const FILESYSTEM_RPC_ERROR_MAP: Partial<[Code.NotFound]: (message: string) => new FileNotFoundError(message),}-function handleFilesystemRpcError(err: unknown): Error {- return handleRpcError(err, FILESYSTEM_RPC_ERROR_MAP)+async function handleFilesystemRpcError(+ err: unknown,+ checkHealth?: SandboxHealthCheck+): Promise<Error> {+ return handleRpcErrorWithHealthCheck(+ err,+ checkHealth,+ FILESYSTEM_RPC_ERROR_MAP+ )}function handleFilesystemEnvdApiError(res: {@@ -331,6 +347,7 @@ export class Filesystem {private readonly defaultWatchTimeout = 60_000 // 60 secondsprivate readonly defaultWatchRecursive = false+ private readonly checkHealth: SandboxHealthCheckconstructor(transport: Transport,@@ -338,6 +355,7 @@ export class Filesystem {private readonly connectionConfig: ConnectionConfig) {this.rpc = createClient(FilesystemService, transport)+ this.checkHealth = () => checkSandboxHealth(this.envdApi)}/**@@ -421,20 +439,24 @@ export class Filesystem {headers['Accept-Encoding'] = 'gzip'}- const res = await this.envdApi.api.GET('/files', {- params: {- query: {- path,- username: user,+ const res = await this.envdApi.api+ .GET('/files', {+ params: {+ query: {+ path,+ username: user,+ },},- },- parseAs: format === 'bytes' ? 'arrayBuffer' : format,- signal: this.connectionConfig.getSignal(- opts?.requestTimeoutMs,- opts?.signal- ),- headers,- })+ parseAs: format === 'bytes' ? 'arrayBuffer' : format,+ signal: this.connectionConfig.getSignal(+ opts?.requestTimeoutMs,+ opts?.signal+ ),+ headers,+ })+ .catch(async (err) => {+ throw await handleEnvdApiFetchError(err, this.checkHealth)+ })const err = await handleFilesystemEnvdApiError(res)if (err) {@@ -565,21 +587,25 @@ export class Filesystem {const filePath = path ?? (file as WriteEntry).pathconst body = await toUploadBody(file.data, useGzip)- const res = await this.envdApi.api.POST('/files', {- params: {- query: {- path: filePath,- username: user,+ const res = await this.envdApi.api+ .POST('/files', {+ params: {+ query: {+ path: filePath,+ username: user,+ },},- },- bodySerializer: () => body,- headers,- signal: this.connectionConfig.getSignal(- writeOpts?.requestTimeoutMs,- writeOpts?.signal- ),- body: {},- })+ bodySerializer: () => body,+ headers,+ signal: this.connectionConfig.getSignal(+ writeOpts?.requestTimeoutMs,+ writeOpts?.signal+ ),+ body: {},+ })+ .catch(async (err) => {+ throw await handleEnvdApiFetchError(err, this.checkHealth)+ })const err = await handleFilesystemEnvdApiError(res)if (err) {@@ -614,21 +640,25 @@ export class Filesystem {)}- const res = await this.envdApi.api.POST('/files', {- params: {- query: {- path,- username: user,+ const res = await this.envdApi.api+ .POST('/files', {+ params: {+ query: {+ path,+ username: user,+ },},- },- bodySerializer: () => formData,- headers: extraHeaders,- signal: this.connectionConfig.getSignal(- writeOpts?.requestTimeoutMs,- writeOpts?.signal- ),- body: {},- })+ bodySerializer: () => formData,+ headers: extraHeaders,+ signal: this.connectionConfig.getSignal(+ writeOpts?.requestTimeoutMs,+ writeOpts?.signal+ ),+ body: {},+ })+ .catch(async (err) => {+ throw await handleEnvdApiFetchError(err, this.checkHealth)+ })const err = await handleFilesystemEnvdApiError(res)if (err) {@@ -713,7 +743,7 @@ export class Filesystem {return entries} catch (err) {- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}@@ -746,7 +776,7 @@ export class Filesystem {}}- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}@@ -786,7 +816,7 @@ export class Filesystem {return mapEntryInfo(entry)} catch (err) {- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}@@ -809,7 +839,7 @@ export class Filesystem {})} catch (err) {- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}@@ -842,7 +872,7 @@ export class Filesystem {}}- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}@@ -878,7 +908,7 @@ export class Filesystem {return mapEntryInfo(res.entry)} catch (err) {- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}@@ -959,10 +989,16 @@ export class Filesystem {await handleWatchDirStartEvent(events)clearStartTimeout()- return new WatchHandle(cleanup, events, onEvent, opts?.onExit)+ return new WatchHandle(+ cleanup,+ events,+ onEvent,+ opts?.onExit,+ this.checkHealth+ )} catch (err) {cleanup()- throw handleFilesystemRpcError(err)+ throw await handleFilesystemRpcError(err, this.checkHealth)}}}diff --git a/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts b/packages/js-sdk/src/sandbox/filesystem/watchHandle.tsindex e29d5703d..9d55573b6 100644--- a/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts+++ b/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts@@ -1,4 +1,7 @@-import { handleRpcError } from '../../envd/rpc'+import {+ handleRpcErrorWithHealthCheck,+ SandboxHealthCheck,+} from '../../envd/rpc'import {EventType,WatchDirResponse,@@ -78,7 +81,8 @@ export class WatchHandle {private readonly handleStop: () => void,private readonly events: AsyncIterable<WatchDirResponse>,private readonly onEvent?: (event: FilesystemEvent) => void | Promise<void>,- private readonly onExit?: (err?: Error) => void | Promise<void>+ private readonly onExit?: (err?: Error) => void | Promise<void>,+ private readonly checkHealth?: SandboxHealthCheck) {this.handleEvents()}@@ -100,7 +104,7 @@ export class WatchHandle {}}} catch (err) {- throw handleRpcError(err)+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)}}diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.tsindex 79d18b98b..1ff75e0ed 100644--- a/packages/js-sdk/src/sandbox/index.ts+++ b/packages/js-sdk/src/sandbox/index.ts@@ -219,12 +219,12 @@ export class Sandbox extends SandboxApi {this.envdApi,this.connectionConfig)- this.commands = new Commands(rpcTransport, this.connectionConfig, {- version: opts.envdVersion,- })- this.pty = new Pty(rpcTransport, this.connectionConfig, {- version: opts.envdVersion,- })+ this.commands = new Commands(+ rpcTransport,+ this.envdApi,+ this.connectionConfig+ )+ this.pty = new Pty(rpcTransport, this.envdApi, this.connectionConfig)this.git = new Git(this.commands)}diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.tsindex e6f63ed50..46b569f21 100644--- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts+++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts@@ -1,5 +1,5 @@import { assert, test, describe } from 'vitest'-import { handleEnvdApiError } from '../../src/envd/api'+import { handleEnvdApiError, handleEnvdApiFetchError } from '../../src/envd/api'import {AuthenticationError,InvalidArgumentError,@@ -102,3 +102,32 @@ describe('handleEnvdApiError', () => {assert.include(err?.message, '500')})})++describe('handleEnvdApiFetchError', () => {+ test('returns the original error for terminated fetch without a health check', async () => {+ const original = new TypeError('terminated')+ const err = await handleEnvdApiFetchError(original)+ assert.strictEqual(err, original)+ })++ test('returns a TimeoutError when the health check says the sandbox is not running', async () => {+ const err = await handleEnvdApiFetchError(+ new TypeError('terminated'),+ async () => false+ )+ assert.instanceOf(err, TimeoutError)+ assert.include(err.message, 'sandbox was killed or reached its end of life')+ })++ test('returns the original error when the health check says the sandbox is running', async () => {+ const original = new TypeError('terminated')+ const err = await handleEnvdApiFetchError(original, async () => true)+ assert.strictEqual(err, original)+ })++ test('returns the original error for other fetch failures', async () => {+ const original = new TypeError('fetch failed')+ const err = await handleEnvdApiFetchError(original)+ assert.strictEqual(err, original)+ })+})diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.tsindex 590ae6a72..4a3f423d8 100644--- a/packages/js-sdk/tests/envd/handleRpcError.test.ts+++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts@@ -1,6 +1,9 @@import { assert, test, describe } from 'vitest'import { Code, ConnectError } from '@connectrpc/connect'-import { handleRpcError } from '../../src/envd/rpc'+import {+ handleRpcError,+ handleRpcErrorWithHealthCheck,+} from '../../src/envd/rpc'import {AuthenticationError,InvalidArgumentError,@@ -44,9 +47,71 @@ describe('handleRpcError', () => {assert.instanceOf(err, SandboxError)})+ test('falls back to generic SandboxError for Unknown "terminated"', () => {+ const err = handleRpcError(new ConnectError('terminated', Code.Unknown))+ assert.instanceOf(err, SandboxError)+ assert.include(err.message, 'terminated')+ assert.notInclude(err.message, 'killed')+ })+test('returns the original error when not a ConnectError', () => {const original = new Error('not connect')const err = handleRpcError(original)assert.strictEqual(err, original)})})++describe('handleRpcErrorWithHealthCheck', () => {+ const terminated = () => new ConnectError('terminated', Code.Unknown)++ test('returns a TimeoutError when the health check says the sandbox is not running', async () => {+ const err = await handleRpcErrorWithHealthCheck(+ terminated(),+ async () => false+ )+ assert.instanceOf(err, TimeoutError)+ assert.include(err.message, 'sandbox was killed or reached its end of life')+ })++ test('falls back to the generic mapping when the health check says the sandbox is running', async () => {+ const err = await handleRpcErrorWithHealthCheck(+ terminated(),+ async () => true+ )+ assert.instanceOf(err, SandboxError)+ assert.notInstanceOf(err, TimeoutError)+ assert.notInclude(err.message, 'killed')+ })++ test('falls back to the generic mapping when the sandbox state is unknown', async () => {+ const err = await handleRpcErrorWithHealthCheck(+ terminated(),+ async () => undefined+ )+ assert.instanceOf(err, SandboxError)+ assert.notInstanceOf(err, TimeoutError)+ assert.notInclude(err.message, 'killed')+ })++ test('falls back to the generic mapping when the health check itself fails', async () => {+ const err = await handleRpcErrorWithHealthCheck(terminated(), async () => {+ throw new Error('health check failed')+ })+ assert.instanceOf(err, SandboxError)+ assert.notInstanceOf(err, TimeoutError)+ assert.notInclude(err.message, 'killed')+ })++ test('does not run the health check for other errors', async () => {+ let called = false+ const err = await handleRpcErrorWithHealthCheck(+ new ConnectError('missing', Code.NotFound),+ async () => {+ called = true+ return false+ }+ )+ assert.instanceOf(err, NotFoundError)+ assert.isFalse(called)+ })+})diff --git a/packages/js-sdk/tests/sandbox/commands/sandboxKilledDuringRun.test.ts b/packages/js-sdk/tests/sandbox/commands/sandboxKilledDuringRun.test.tsnew file mode 100644index 000000000..511a7a38a--- /dev/null+++ b/packages/js-sdk/tests/sandbox/commands/sandboxKilledDuringRun.test.ts@@ -0,0 +1,20 @@+import { expect } from 'vitest'+import { TimeoutError } from '../../../src/index.js'++import { sandboxTest, isDebug } from '../../setup.js'++sandboxTest.skipIf(isDebug)(+ 'killing the sandbox while a command is running throws an actionable error',+ async ({ sandbox }) => {+ const cmd = await sandbox.commands.run('sleep 60', { background: true })++ await sandbox.kill()++ const err = await cmd.wait().catch((e) => e)+ expect(err).toBeInstanceOf(TimeoutError)+ // The health check confirms the sandbox is gone, so the error states it outright+ expect(err.message).toContain(+ 'sandbox was killed or reached its end of life'+ )+ }+)diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.pyindex 7074f9e6a..811d58d59 100644--- a/packages/python-sdk/e2b/envd/api.py+++ b/packages/python-sdk/e2b/envd/api.py@@ -3,6 +3,7 @@ import jsonfrom typing import Callable, Optional+from e2b.envd.rpc import format_terminated_exceptionfrom e2b.exceptions import (SandboxException,NotFoundException,@@ -29,6 +30,85 @@ _DEFAULT_API_ERROR_MAP: dict[int, Callable[[str], Exception]] = {}+HEALTH_CHECK_TIMEOUT = 5 # seconds+++def check_sandbox_health(envd_api: httpx.Client) -> Optional[bool]:+ """Probe the sandbox's envd health endpoint.++ :return: ``True`` if the sandbox is running, ``False`` if it is not, ``None`` if its state could not be determined.+ """+ try:+ r = envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT)+ if r.status_code == 502:+ return False+ if r.is_success:+ return True+ return None+ except Exception:+ return None+++async def acheck_sandbox_health(envd_api: httpx.AsyncClient) -> Optional[bool]:+ """Async version of :func:`check_sandbox_health`."""+ try:+ r = await envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT)+ if r.status_code == 502:+ return False+ if r.is_success:+ return True+ return None+ except Exception:+ return None+++def handle_envd_api_transport_exception(+ e: Exception,+ sandbox_running: Optional[bool] = None,+) -> Exception:+ """Handle transport-level errors from envd API requests.++ :param e: The caught exception, expected to be a transport-level ``httpx`` error.+ :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request.+ :return: A ``TimeoutException`` when the connection dropped mid-request and the sandbox is confirmed gone, or the original exception unchanged otherwise.+ """+ # A remote protocol error (e.g. an HTTP/2 stream reset) means the connection to the+ # sandbox was dropped mid-request — either the sandbox died or the network failed+ if isinstance(e, httpx.RemoteProtocolError):+ return format_terminated_exception(e, sandbox_running)++ return e+++def handle_envd_api_transport_exception_with_health(+ e: Exception,+ envd_api: httpx.Client,+) -> Exception:+ """Like :func:`handle_envd_api_transport_exception`, but when the connection to the+ sandbox was dropped mid-request it probes the sandbox health to tell apart the sandbox+ being killed from a transient network failure (e.g. a load balancer dropping the connection).+ """+ sandbox_running = (+ check_sandbox_health(envd_api)+ if isinstance(e, httpx.RemoteProtocolError)+ else None+ )+ return handle_envd_api_transport_exception(e, sandbox_running)+++async def ahandle_envd_api_transport_exception_with_health(+ e: Exception,+ envd_api: httpx.AsyncClient,+) -> Exception:+ """Async version of :func:`handle_envd_api_transport_exception_with_health`."""+ sandbox_running = (+ await acheck_sandbox_health(envd_api)+ if isinstance(e, httpx.RemoteProtocolError)+ else None+ )+ return handle_envd_api_transport_exception(e, sandbox_running)++def get_message(e: httpx.Response) -> str:try:message = e.json().get("message", e.text)diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.pyindex f8d263c5e..72ace254d 100644--- a/packages/python-sdk/e2b/envd/rpc.py+++ b/packages/python-sdk/e2b/envd/rpc.py@@ -1,6 +1,7 @@import base64-from typing import Callable, Optional+import httpcore+from typing import Awaitable, Callable, Optionalfrom packaging.version import Versionfrom e2b_connect.client import Code, ConnectException@@ -33,15 +34,31 @@ _DEFAULT_RPC_ERROR_MAP: dict[Code, Callable[[str], Exception]] = {}+def format_terminated_exception(+ e: Exception,+ sandbox_running: Optional[bool],+) -> Exception:+ """Handle an exception for a connection to the sandbox dropped mid-request: when a+ sandbox health probe confirmed the sandbox is gone (``sandbox_running is False``),+ return a ``TimeoutException``; otherwise return the original error unchanged."""+ if sandbox_running is False:+ return TimeoutException(+ f"{e}: The sandbox was killed or reached its end of life while the request was in flight."+ )+ return e++def handle_rpc_exception(e: Exception,error_map: Optional[dict[Code, Callable[[str], Exception]]] = None,+ sandbox_running: Optional[bool] = None,):"""Handle errors from envd RPC calls by mapping gRPC status codes to specific exception types.- :param e: The caught exception, expected to be a ``ConnectException``.+ :param e: The caught exception, expected to be a ``ConnectException`` or a transport-level ``httpcore`` error.:param error_map: Optional map of gRPC codes to exception factories that override the defaults.- :return: The corresponding exception, or the original exception if not a ``ConnectException``.+ :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request.+ :return: The corresponding exception. A connection dropped mid-request with the sandbox confirmed gone becomes a ``TimeoutException``; non-``ConnectException`` errors are otherwise returned as-is."""if isinstance(e, ConnectException):if error_map and e.status in error_map:@@ -51,8 +68,46 @@ def handle_rpc_exception(return _DEFAULT_RPC_ERROR_MAP[e.status](e.message)return SandboxException(f"{e.status}: {e.message}")- else:- return e++ # A remote protocol error (e.g. an HTTP/2 stream reset) means the connection to the+ # sandbox was dropped mid-request — either the sandbox died or the network failed+ if isinstance(e, httpcore.RemoteProtocolError):+ return format_terminated_exception(e, sandbox_running)++ return e+++def handle_rpc_exception_with_health(+ e: Exception,+ check_health: Optional[Callable[[], Optional[bool]]] = None,+ error_map: Optional[dict[Code, Callable[[str], Exception]]] = None,+):+ """Like :func:`handle_rpc_exception`, but when the connection to the sandbox was+ dropped mid-request it probes the sandbox health to tell apart the sandbox being+ killed from a transient network failure (e.g. a load balancer dropping the connection).+ """+ sandbox_running = None+ if check_health is not None and isinstance(e, httpcore.RemoteProtocolError):+ try:+ sandbox_running = check_health()+ except Exception:+ sandbox_running = None+ return handle_rpc_exception(e, error_map, sandbox_running)+++async def ahandle_rpc_exception_with_health(+ e: Exception,+ check_health: Optional[Callable[[], Awaitable[Optional[bool]]]] = None,+ error_map: Optional[dict[Code, Callable[[str], Exception]]] = None,+):+ """Async version of :func:`handle_rpc_exception_with_health`."""+ sandbox_running = None+ if check_health is not None and isinstance(e, httpcore.RemoteProtocolError):+ try:+ sandbox_running = await check_health()+ except Exception:+ sandbox_running = None+ return handle_rpc_exception(e, error_map, sandbox_running)def authentication_header(diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.pyindex 1280d0fc8..8142d5321 100644--- a/packages/python-sdk/e2b/sandbox_async/commands/command.py+++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py@@ -2,6 +2,7 @@ from typing import Dict, List, Literal, Optional, Union, overloadimport e2b_connectimport httpcore+import httpxfrom packaging.version import Versionfrom e2b.connection_config import (ConnectionConfig,@@ -10,7 +11,8 @@ from e2b.connection_config import (KEEPALIVE_PING_INTERVAL_SEC,)from e2b.envd.process import process_connect, process_pb2-from e2b.envd.rpc import authentication_header, handle_rpc_exception+from e2b.envd.api import acheck_sandbox_health+from e2b.envd.rpc import authentication_header, ahandle_rpc_exception_with_healthfrom e2b.envd.versions import ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSEfrom e2b.exceptions import SandboxExceptionfrom e2b.sandbox.commands.main import ProcessInfo@@ -30,9 +32,11 @@ class Commands:connection_config: ConnectionConfig,pool: httpcore.AsyncConnectionPool,envd_version: Version,+ envd_api: httpx.AsyncClient,) -> None:self._connection_config = connection_configself._envd_version = envd_version+ self._check_health = lambda: acheck_sandbox_health(envd_api)self._rpc = process_connect.ProcessClient(envd_api_url,# TODO: Fix and enable compression again — the headers compression is not solved for streaming.@@ -72,7 +76,7 @@ class Commands:for p in res.processes]except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def kill(self,@@ -103,7 +107,7 @@ class Commands:if isinstance(e, e2b_connect.ConnectException):if e.status == e2b_connect.Code.not_found:return False- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def send_stdin(self,@@ -131,7 +135,7 @@ class Commands:),)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def close_stdin(self,@@ -162,7 +166,7 @@ class Commands:),)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)@overloadasync def run(@@ -318,9 +322,10 @@ class Commands:handle_close_stdin=lambda request_timeout=None: self.close_stdin(pid, request_timeout),+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def connect(self,@@ -376,6 +381,7 @@ class Commands:handle_close_stdin=lambda request_timeout=None: self.close_stdin(pid, request_timeout),+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.pyindex e74ab6ce7..7b094cd6f 100644--- a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py+++ b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py@@ -5,12 +5,13 @@ from typing import (Callable,Any,AsyncGenerator,+ Awaitable,Union,Tuple,Coroutine,)-from e2b.envd.rpc import handle_rpc_exception+from e2b.envd.rpc import ahandle_rpc_exception_with_healthfrom e2b.envd.process import process_pb2from e2b.exceptions import SandboxExceptionfrom e2b.sandbox.commands.command_handle import (@@ -89,11 +90,13 @@ class AsyncCommandHandle:handle_close_stdin: Optional[Callable[[Optional[float]], Coroutine[Any, Any, None]]] = None,+ check_health: Optional[Callable[[], Awaitable[Optional[bool]]]] = None,):self._pid = pidself._handle_kill = handle_killself._handle_send_stdin = handle_send_stdinself._handle_close_stdin = handle_close_stdin+ self._check_health = check_healthself._events = eventsself._stdout: str = ""@@ -170,7 +173,9 @@ class AsyncCommandHandle:except StopAsyncIteration:passexcept Exception as e:- self._iteration_exception = handle_rpc_exception(e)+ self._iteration_exception = await ahandle_rpc_exception_with_health(+ e, self._check_health+ )async def wait(self) -> CommandResult:"""diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.pyindex 5b7b69aef..809f061b8 100644--- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py+++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py@@ -2,6 +2,7 @@ from typing import Dict, Optionalimport e2b_connectimport httpcore+import httpxfrom packaging.version import Versionfrom e2b.envd.process import process_connect, process_pb2@@ -12,7 +13,8 @@ from e2b.connection_config import (KEEPALIVE_PING_INTERVAL_SEC,)from e2b.exceptions import SandboxException-from e2b.envd.rpc import authentication_header, handle_rpc_exception+from e2b.envd.api import acheck_sandbox_health+from e2b.envd.rpc import authentication_header, ahandle_rpc_exception_with_healthfrom e2b.sandbox.commands.command_handle import PtySizefrom e2b.sandbox_async.commands.command_handle import (AsyncCommandHandle,@@ -32,9 +34,11 @@ class Pty:connection_config: ConnectionConfig,pool: httpcore.AsyncConnectionPool,envd_version: Version,+ envd_api: httpx.AsyncClient,) -> None:self._connection_config = connection_configself._envd_version = envd_version+ self._check_health = lambda: acheck_sandbox_health(envd_api)self._rpc = process_connect.ProcessClient(envd_api_url,# TODO: Fix and enable compression again — the headers compression is not solved for streaming.@@ -72,7 +76,7 @@ class Pty:if isinstance(e, e2b_connect.ConnectException):if e.status == e2b_connect.Code.not_found:return False- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def send_stdin(self,@@ -100,7 +104,7 @@ class Pty:),)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def create(self,@@ -164,9 +168,10 @@ class Pty:handle_kill=lambda: self.kill(start_event.event.start.pid),events=events,on_pty=on_data,+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def connect(self,@@ -211,9 +216,10 @@ class Pty:handle_kill=lambda: self.kill(start_event.event.start.pid),events=events,on_pty=on_data,+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def resize(self,diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.pyindex 6c88811b0..208a08c0f 100644--- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py@@ -14,9 +14,14 @@ from e2b.connection_config import (Username,default_username,)-from e2b.envd.api import ENVD_API_FILES_ROUTE, ahandle_envd_api_exception+from e2b.envd.api import (+ ENVD_API_FILES_ROUTE,+ acheck_sandbox_health,+ ahandle_envd_api_exception,+ ahandle_envd_api_transport_exception_with_health,+)from e2b.envd.filesystem import filesystem_connect, filesystem_pb2-from e2b.envd.rpc import authentication_header, handle_rpc_exception+from e2b.envd.rpc import authentication_header, ahandle_rpc_exception_with_healthfrom e2b.envd.versions import (ENVD_DEFAULT_USER,ENVD_FILE_METADATA,@@ -56,8 +61,12 @@ _FILESYSTEM_HTTP_ERROR_MAP = {}-def _handle_filesystem_rpc_exception(e: Exception) -> Exception:- return handle_rpc_exception(e, _FILESYSTEM_RPC_ERROR_MAP)+async def _ahandle_filesystem_rpc_exception(+ e: Exception, envd_api: httpx.AsyncClient+) -> Exception:+ return await ahandle_rpc_exception_with_health(+ e, lambda: acheck_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP+ )async def _ahandle_filesystem_envd_api_exception(r):@@ -178,12 +187,17 @@ class Filesystem:if gzip:headers["Accept-Encoding"] = "gzip"- r = await self._envd_api.get(- ENVD_API_FILES_ROUTE,- params=params,- headers=headers,- timeout=self._connection_config.get_request_timeout(request_timeout),- )+ try:+ r = await self._envd_api.get(+ ENVD_API_FILES_ROUTE,+ params=params,+ headers=headers,+ timeout=self._connection_config.get_request_timeout(request_timeout),+ )+ except httpx.RemoteProtocolError as e:+ raise await ahandle_envd_api_transport_exception_with_health(+ e, self._envd_api+ )err = await _ahandle_filesystem_envd_api_exception(r)if err:@@ -295,15 +309,20 @@ class Filesystem:if gzip:headers["Content-Encoding"] = "gzip"- r = await self._envd_api.post(- ENVD_API_FILES_ROUTE,- content=to_upload_body(file_data, gzip),- headers=headers,- params=params,- timeout=self._connection_config.get_request_timeout(- request_timeout- ),- )+ try:+ r = await self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ content=to_upload_body(file_data, gzip),+ headers=headers,+ params=params,+ timeout=self._connection_config.get_request_timeout(+ request_timeout+ ),+ )+ except httpx.RemoteProtocolError as e:+ raise await ahandle_envd_api_transport_exception_with_health(+ e, self._envd_api+ )err = await _ahandle_filesystem_envd_api_exception(r)if err:@@ -335,13 +354,20 @@ class Filesystem:if len(httpx_files) == 0:return []- r = await self._envd_api.post(- ENVD_API_FILES_ROUTE,- files=httpx_files,- params=params,- headers=extra_headers,- timeout=self._connection_config.get_request_timeout(request_timeout),- )+ try:+ r = await self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ files=httpx_files,+ params=params,+ headers=extra_headers,+ timeout=self._connection_config.get_request_timeout(+ request_timeout+ ),+ )+ except httpx.RemoteProtocolError as e:+ raise await ahandle_envd_api_transport_exception_with_health(+ e, self._envd_api+ )err = await _ahandle_filesystem_envd_api_exception(r)if err:@@ -395,7 +421,7 @@ class Filesystem:return entriesexcept Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)async def exists(self,@@ -427,7 +453,7 @@ class Filesystem:if isinstance(e, connect.ConnectException):if e.status == connect.Code.not_found:return False- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)async def get_info(self,@@ -455,7 +481,7 @@ class Filesystem:return map_entry_info(r.entry)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)async def remove(self,@@ -479,7 +505,7 @@ class Filesystem:headers=authentication_header(self._envd_version, user),)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)async def rename(self,@@ -512,7 +538,7 @@ class Filesystem:return map_entry_info(r.entry)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)async def make_dir(self,@@ -543,7 +569,7 @@ class Filesystem:if isinstance(e, connect.ConnectException):if e.status == connect.Code.already_exists:return False- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)async def watch_dir(self,@@ -615,6 +641,11 @@ class Filesystem:f"Failed to start watch: expected start event, got {start_event}",)- return AsyncWatchHandle(events=events, on_event=on_event, on_exit=on_exit)+ return AsyncWatchHandle(+ events=events,+ on_event=on_event,+ on_exit=on_exit,+ check_health=lambda: acheck_sandbox_health(self._envd_api),+ )except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py b/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.pyindex a33d13fd1..092b066f3 100644--- a/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py@@ -1,9 +1,9 @@import asyncioimport inspect-from typing import Any, AsyncGenerator, Optional+from typing import Any, AsyncGenerator, Awaitable, Callable, Optional-from e2b.envd.rpc import handle_rpc_exception+from e2b.envd.rpc import ahandle_rpc_exception_with_healthfrom e2b.envd.filesystem.filesystem_pb2 import WatchDirResponsefrom e2b.sandbox.filesystem.filesystem import map_entry_infofrom e2b.sandbox.filesystem.watch_handle import FilesystemEvent, map_event_type@@ -22,10 +22,12 @@ class AsyncWatchHandle:events: AsyncGenerator[WatchDirResponse, Any],on_event: OutputHandler[FilesystemEvent],on_exit: Optional[OutputHandler[Exception]] = None,+ check_health: Optional[Callable[[], Awaitable[Optional[bool]]]] = None,):self._events = eventsself._on_event = on_eventself._on_exit = on_exit+ self._check_health = check_healthself._wait = asyncio.create_task(self._handle_events())@@ -56,7 +58,7 @@ class AsyncWatchHandle:),)except Exception as e:- raise handle_rpc_exception(e)+ raise await ahandle_rpc_exception_with_health(e, self._check_health)async def _handle_events(self):try:diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.pyindex a09ea3336..a4c5d9b7d 100644--- a/packages/python-sdk/e2b/sandbox_async/main.py+++ b/packages/python-sdk/e2b/sandbox_async/main.py@@ -122,12 +122,14 @@ class AsyncSandbox(SandboxApi):self.connection_config,self._transport.pool,self._envd_version,+ self._envd_api,)self._pty = Pty(self.envd_api_url,self.connection_config,self._transport.pool,self._envd_version,+ self._envd_api,)self._git = Git(self._commands)diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.pyindex e2252bf4a..0f0585ad8 100644--- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py@@ -2,6 +2,7 @@ import threadingfrom typing import Callable, Dict, List, Literal, Optional, Union, overloadimport e2b_connect+import httpxfrom packaging.version import Versionfrom e2b.api.client_sync import get_envd_transportfrom e2b.connection_config import (@@ -11,7 +12,8 @@ from e2b.connection_config import (KEEPALIVE_PING_INTERVAL_SEC,)from e2b.envd.process import process_connect, process_pb2-from e2b.envd.rpc import authentication_header, handle_rpc_exception+from e2b.envd.api import check_sandbox_health+from e2b.envd.rpc import authentication_header, handle_rpc_exception_with_healthfrom e2b.envd.versions import ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSEfrom e2b.exceptions import SandboxExceptionfrom e2b.sandbox.commands.main import ProcessInfo@@ -35,6 +37,14 @@ class Commands:self._envd_version = envd_versionself._thread_local = threading.local()+ def _create_envd_api(self) -> httpx.Client:+ transport = get_envd_transport(self._connection_config)+ return httpx.Client(+ base_url=self._envd_api_url,+ transport=transport,+ headers=self._connection_config.sandbox_headers,+ )+def _create_rpc(self) -> process_connect.ProcessClient:transport = get_envd_transport(self._connection_config)return process_connect.ProcessClient(@@ -46,6 +56,14 @@ class Commands:headers=self._connection_config.sandbox_headers,)+ @property+ def _envd_api(self) -> httpx.Client:+ envd_api = getattr(self._thread_local, "envd_api", None)+ if envd_api is None:+ envd_api = self._create_envd_api()+ self._thread_local.envd_api = envd_api+ return envd_api+@propertydef _rpc(self) -> process_connect.ProcessClient:rpc = getattr(self._thread_local, "rpc", None)@@ -54,6 +72,9 @@ class Commands:self._thread_local.rpc = rpcreturn rpc+ def _check_health(self) -> Optional[bool]:+ return check_sandbox_health(self._envd_api)+def list(self,request_timeout: Optional[float] = None,@@ -84,7 +105,7 @@ class Commands:for p in res.processes]except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def kill(self,@@ -115,7 +136,7 @@ class Commands:if isinstance(e, e2b_connect.ConnectException):if e.status == e2b_connect.Code.not_found:return False- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def send_stdin(self,@@ -143,7 +164,7 @@ class Commands:),)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def close_stdin(self,@@ -174,7 +195,7 @@ class Commands:),)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)@overloaddef run(@@ -329,9 +350,10 @@ class Commands:handle_close_stdin=lambda request_timeout=None: self.close_stdin(pid, request_timeout),+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def connect(self,@@ -381,6 +403,7 @@ class Commands:handle_close_stdin=lambda request_timeout=None: self.close_stdin(pid, request_timeout),+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.pyindex f9b4f6af0..27138b9d8 100644--- a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py@@ -1,6 +1,6 @@from typing import Optional, Callable, Any, Generator, Union, Tuple-from e2b.envd.rpc import handle_rpc_exception+from e2b.envd.rpc import handle_rpc_exception_with_healthfrom e2b.envd.process import process_pb2from e2b.exceptions import SandboxExceptionfrom e2b.sandbox.commands.command_handle import (@@ -37,11 +37,13 @@ class CommandHandle:Callable[[Union[str, bytes], Optional[float]], None]] = None,handle_close_stdin: Optional[Callable[[Optional[float]], None]] = None,+ check_health: Optional[Callable[[], Optional[bool]]] = None,):self._pid = pidself._handle_kill = handle_killself._handle_send_stdin = handle_send_stdinself._handle_close_stdin = handle_close_stdin+ self._check_health = check_healthself._events = eventsself._stdout: str = ""@@ -90,7 +92,7 @@ class CommandHandle:error=event.event.end.error,)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def disconnect(self) -> None:"""@@ -128,7 +130,9 @@ class CommandHandle:except StopIteration:passexcept Exception as e:- self._iteration_exception = handle_rpc_exception(e)+ self._iteration_exception = handle_rpc_exception_with_health(+ e, self._check_health+ )if self._iteration_exception:raise self._iteration_exceptiondiff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.pyindex 34fc7bfcc..2abf0ed85 100644--- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py+++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py@@ -1,4 +1,5 @@import e2b_connect+import httpximport threadingfrom typing import Dict, Optional@@ -13,7 +14,8 @@ from e2b.connection_config import (KEEPALIVE_PING_INTERVAL_SEC,)from e2b.exceptions import SandboxException-from e2b.envd.rpc import authentication_header, handle_rpc_exception+from e2b.envd.api import check_sandbox_health+from e2b.envd.rpc import authentication_header, handle_rpc_exception_with_healthfrom e2b.sandbox.commands.command_handle import PtySizefrom e2b.sandbox_sync.commands.command_handle import CommandHandle@@ -34,6 +36,14 @@ class Pty:self._envd_version = envd_versionself._thread_local = threading.local()+ def _create_envd_api(self) -> httpx.Client:+ transport = get_envd_transport(self._connection_config)+ return httpx.Client(+ base_url=self._envd_api_url,+ transport=transport,+ headers=self._connection_config.sandbox_headers,+ )+def _create_rpc(self) -> process_connect.ProcessClient:transport = get_envd_transport(self._connection_config)return process_connect.ProcessClient(@@ -45,6 +55,14 @@ class Pty:headers=self._connection_config.sandbox_headers,)+ @property+ def _envd_api(self) -> httpx.Client:+ envd_api = getattr(self._thread_local, "envd_api", None)+ if envd_api is None:+ envd_api = self._create_envd_api()+ self._thread_local.envd_api = envd_api+ return envd_api+@propertydef _rpc(self) -> process_connect.ProcessClient:rpc = getattr(self._thread_local, "rpc", None)@@ -53,6 +71,9 @@ class Pty:self._thread_local.rpc = rpcreturn rpc+ def _check_health(self) -> Optional[bool]:+ return check_sandbox_health(self._envd_api)+def kill(self,pid: int,@@ -81,7 +102,7 @@ class Pty:if isinstance(e, e2b_connect.ConnectException):if e.status == e2b_connect.Code.not_found:return False- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def send_stdin(self,@@ -109,7 +130,7 @@ class Pty:),)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def create(self,@@ -170,9 +191,10 @@ class Pty:pid=start_event.event.start.pid,handle_kill=lambda: self.kill(start_event.event.start.pid),events=events,+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def connect(self,@@ -214,9 +236,10 @@ class Pty:pid=start_event.event.start.pid,handle_kill=lambda: self.kill(start_event.event.start.pid),events=events,+ check_health=self._check_health,)except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)def resize(self,diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.pyindex 7a233c1f8..d43767bf5 100644--- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py+++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py@@ -15,9 +15,14 @@ from e2b.connection_config import ()from e2b_connect.client import Code-from e2b.envd.api import ENVD_API_FILES_ROUTE, handle_envd_api_exception+from e2b.envd.api import (+ ENVD_API_FILES_ROUTE,+ check_sandbox_health,+ handle_envd_api_exception,+ handle_envd_api_transport_exception_with_health,+)from e2b.envd.filesystem import filesystem_connect, filesystem_pb2-from e2b.envd.rpc import authentication_header, handle_rpc_exception+from e2b.envd.rpc import authentication_header, handle_rpc_exception_with_healthfrom e2b.envd.versions import (ENVD_DEFAULT_USER,ENVD_FILE_METADATA,@@ -55,8 +60,10 @@ _FILESYSTEM_HTTP_ERROR_MAP = {}-def _handle_filesystem_rpc_exception(e: Exception) -> Exception:- return handle_rpc_exception(e, _FILESYSTEM_RPC_ERROR_MAP)+def _handle_filesystem_rpc_exception(e: Exception, envd_api: httpx.Client) -> Exception:+ return handle_rpc_exception_with_health(+ e, lambda: check_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP+ )def _handle_filesystem_envd_api_exception(r):@@ -200,12 +207,15 @@ class Filesystem:if gzip:headers["Accept-Encoding"] = "gzip"- r = self._envd_api.get(- ENVD_API_FILES_ROUTE,- params=params,- headers=headers,- timeout=self._connection_config.get_request_timeout(request_timeout),- )+ try:+ r = self._envd_api.get(+ ENVD_API_FILES_ROUTE,+ params=params,+ headers=headers,+ timeout=self._connection_config.get_request_timeout(request_timeout),+ )+ except httpx.RemoteProtocolError as e:+ raise handle_envd_api_transport_exception_with_health(e, self._envd_api)err = _handle_filesystem_envd_api_exception(r)if err:@@ -316,15 +326,20 @@ class Filesystem:if gzip:headers["Content-Encoding"] = "gzip"- r = self._envd_api.post(- ENVD_API_FILES_ROUTE,- content=to_upload_body(file_data, gzip),- headers=headers,- params=params,- timeout=self._connection_config.get_request_timeout(- request_timeout- ),- )+ try:+ r = self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ content=to_upload_body(file_data, gzip),+ headers=headers,+ params=params,+ timeout=self._connection_config.get_request_timeout(+ request_timeout+ ),+ )+ except httpx.RemoteProtocolError as e:+ raise handle_envd_api_transport_exception_with_health(+ e, self._envd_api+ )err = _handle_filesystem_envd_api_exception(r)if err:@@ -350,13 +365,18 @@ class Filesystem:if len(httpx_files) == 0:return []- r = self._envd_api.post(- ENVD_API_FILES_ROUTE,- files=httpx_files,- params=params,- headers=extra_headers,- timeout=self._connection_config.get_request_timeout(request_timeout),- )+ try:+ r = self._envd_api.post(+ ENVD_API_FILES_ROUTE,+ files=httpx_files,+ params=params,+ headers=extra_headers,+ timeout=self._connection_config.get_request_timeout(+ request_timeout+ ),+ )+ except httpx.RemoteProtocolError as e:+ raise handle_envd_api_transport_exception_with_health(e, self._envd_api)err = _handle_filesystem_envd_api_exception(r)if err:@@ -410,7 +430,7 @@ class Filesystem:return entriesexcept Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)def exists(self,@@ -441,7 +461,7 @@ class Filesystem:if isinstance(e, e2b_connect.ConnectException):if e.status == e2b_connect.Code.not_found:return False- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)def get_info(self,@@ -469,7 +489,7 @@ class Filesystem:return map_entry_info(r.entry)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)def remove(self,@@ -493,7 +513,7 @@ class Filesystem:headers=authentication_header(self._envd_version, user),)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)def rename(self,@@ -526,7 +546,7 @@ class Filesystem:return map_entry_info(r.entry)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)def make_dir(self,@@ -557,7 +577,7 @@ class Filesystem:if isinstance(e, e2b_connect.ConnectException):if e.status == e2b_connect.Code.already_exists:return False- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)def watch_dir(self,@@ -615,6 +635,10 @@ class Filesystem:},)except Exception as e:- raise _handle_filesystem_rpc_exception(e)+ raise _handle_filesystem_rpc_exception(e, self._envd_api)- return WatchHandle(lambda: self._rpc, r.watcher_id)+ return WatchHandle(+ lambda: self._rpc,+ r.watcher_id,+ lambda: check_sandbox_health(self._envd_api),+ )diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.pyindex 4b47b0072..84370e013 100644--- a/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py+++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py@@ -1,4 +1,4 @@-from typing import Callable, List+from typing import Callable, List, Optionalfrom e2b import SandboxExceptionfrom e2b.envd.filesystem import filesystem_connect@@ -6,7 +6,7 @@ from e2b.envd.filesystem.filesystem_pb2 import (GetWatcherEventsRequest,RemoveWatcherRequest,)-from e2b.envd.rpc import handle_rpc_exception+from e2b.envd.rpc import handle_rpc_exception_with_healthfrom e2b.sandbox.filesystem.filesystem import map_entry_infofrom e2b.sandbox.filesystem.watch_handle import FilesystemEvent, map_event_type@@ -23,9 +23,11 @@ class WatchHandle:self,get_rpc: Callable[[], filesystem_connect.FilesystemClient],watcher_id: str,+ check_health: Optional[Callable[[], Optional[bool]]] = None,):self._get_rpc = get_rpcself._watcher_id = watcher_id+ self._check_health = check_healthself._closed = Falsedef stop(self):@@ -38,7 +40,7 @@ class WatchHandle:RemoveWatcherRequest(watcher_id=self._watcher_id))except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)self._closed = True@@ -56,7 +58,7 @@ class WatchHandle:GetWatcherEventsRequest(watcher_id=self._watcher_id))except Exception as e:- raise handle_rpc_exception(e)+ raise handle_rpc_exception_with_health(e, self._check_health)events = []for event in r.events:diff --git a/packages/python-sdk/e2b_connect/client.py b/packages/python-sdk/e2b_connect/client.pyindex 7e1ced394..e32fa7de5 100644--- a/packages/python-sdk/e2b_connect/client.py+++ b/packages/python-sdk/e2b_connect/client.py@@ -361,6 +361,8 @@ class Client:},}+ # Note: no retry here — generator functions don't execute until iterated, so a+ # call-level retry never fires, and retrying mid-stream would replay delivered events.async def acall_server_stream(self,req,diff --git a/packages/python-sdk/tests/async/sandbox_async/commands/test_sandbox_killed_during_run.py b/packages/python-sdk/tests/async/sandbox_async/commands/test_sandbox_killed_during_run.pynew file mode 100644index 000000000..2f83f210b--- /dev/null+++ b/packages/python-sdk/tests/async/sandbox_async/commands/test_sandbox_killed_during_run.py@@ -0,0 +1,16 @@+import pytest++from e2b import AsyncSandbox, TimeoutException++[email protected]_debug()+async def test_kill_sandbox_while_command_is_running(async_sandbox: AsyncSandbox):+ cmd = await async_sandbox.commands.run("sleep 60", background=True)++ await async_sandbox.kill()++ with pytest.raises(TimeoutException) as exc_info:+ await cmd.wait()++ # The health check confirms the sandbox is gone, so the error states it outright+ assert "sandbox was killed or reached its end of life" in str(exc_info.value)diff --git a/packages/python-sdk/tests/sync/sandbox_sync/commands/test_sandbox_killed_during_run.py b/packages/python-sdk/tests/sync/sandbox_sync/commands/test_sandbox_killed_during_run.pynew file mode 100644index 000000000..81dd1cb28--- /dev/null+++ b/packages/python-sdk/tests/sync/sandbox_sync/commands/test_sandbox_killed_during_run.py@@ -0,0 +1,16 @@+import pytest++from e2b import Sandbox, TimeoutException++[email protected]_debug()+def test_kill_sandbox_while_command_is_running(sandbox: Sandbox):+ cmd = sandbox.commands.run("sleep 60", background=True)++ sandbox.kill()++ with pytest.raises(TimeoutException) as exc_info:+ cmd.wait()++ # The health check confirms the sandbox is gone, so the error states it outright+ assert "sandbox was killed or reached its end of life" in str(exc_info.value)diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.pyindex 5c501391c..7c0694da7 100644--- a/packages/python-sdk/tests/test_envd_api_exception.py+++ b/packages/python-sdk/tests/test_envd_api_exception.py@@ -1,4 +1,9 @@-from e2b.envd.api import format_envd_api_exception+import httpx++from e2b.envd.api import (+ format_envd_api_exception,+ handle_envd_api_transport_exception,+)from e2b.exceptions import (AuthenticationException,InvalidArgumentException,@@ -45,3 +50,35 @@ def test_falls_back_to_sandbox_exception():err = format_envd_api_exception(500, "Internal error")assert isinstance(err, SandboxException)assert "500" in str(err)+++def test_returns_raw_remote_protocol_error_without_health_result():+ original = httpx.RemoteProtocolError("peer closed connection")+ err = handle_envd_api_transport_exception(original)+ assert err is original+++def test_returns_raw_network_errors():+ original = httpx.ReadError("read failed")+ err = handle_envd_api_transport_exception(original)+ assert err is original+++def test_returns_original_when_not_transport_error():+ original = ValueError("not transport")+ err = handle_envd_api_transport_exception(original)+ assert err is original+++def test_health_result_confirms_sandbox_killed():+ err = handle_envd_api_transport_exception(+ httpx.RemoteProtocolError("peer closed connection"), sandbox_running=False+ )+ assert isinstance(err, TimeoutException)+ assert "sandbox was killed or reached its end of life" in str(err)+++def test_health_result_running_returns_raw_error():+ original = httpx.RemoteProtocolError("peer closed connection")+ err = handle_envd_api_transport_exception(original, sandbox_running=True)+ assert err is originaldiff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.pyindex 42a455ee2..71109d3da 100644--- a/packages/python-sdk/tests/test_envd_rpc_exception.py+++ b/packages/python-sdk/tests/test_envd_rpc_exception.py@@ -1,5 +1,11 @@+import httpcore+from e2b_connect.client import Code, ConnectException-from e2b.envd.rpc import handle_rpc_exception+from e2b.envd.rpc import (+ ahandle_rpc_exception_with_health,+ handle_rpc_exception,+ handle_rpc_exception_with_health,+)from e2b.exceptions import (AuthenticationException,InvalidArgumentException,@@ -41,7 +47,82 @@ def test_falls_back_to_sandbox_exception():assert isinstance(err, SandboxException)+def test_returns_raw_remote_protocol_error_without_health_result():+ original = httpcore.RemoteProtocolError(+ "<StreamReset stream_id:1, error_code:2, remote_reset:True>"+ )+ err = handle_rpc_exception(original)+ assert err is original+++def test_returns_raw_network_errors():+ original = httpcore.ReadError("read failed")+ err = handle_rpc_exception(original)+ assert err is original++def test_returns_original_when_not_connect_exception():original = ValueError("not connect")err = handle_rpc_exception(original)assert err is original+++def _stream_reset() -> httpcore.RemoteProtocolError:+ return httpcore.RemoteProtocolError(+ "<StreamReset stream_id:1, error_code:2, remote_reset:True>"+ )+++def test_health_check_confirms_sandbox_killed():+ err = handle_rpc_exception_with_health(_stream_reset(), lambda: False)+ assert isinstance(err, TimeoutException)+ assert "sandbox was killed or reached its end of life" in str(err)+++def test_health_check_running_returns_raw_error():+ original = _stream_reset()+ err = handle_rpc_exception_with_health(original, lambda: True)+ assert err is original+++def test_health_check_unknown_returns_raw_error():+ original = _stream_reset()+ err = handle_rpc_exception_with_health(original, lambda: None)+ assert err is original+++def test_health_check_failure_returns_raw_error():+ def check():+ raise RuntimeError("health check failed")++ original = _stream_reset()+ err = handle_rpc_exception_with_health(original, check)+ assert err is original+++def test_health_check_not_run_for_other_exceptions():+ def fail():+ raise AssertionError("health check should not run")++ err = handle_rpc_exception_with_health(+ ConnectException(Code.not_found, "missing"), fail+ )+ assert isinstance(err, NotFoundException)+++async def test_async_health_check_confirms_sandbox_killed():+ async def check():+ return False++ err = await ahandle_rpc_exception_with_health(_stream_reset(), check)+ assert isinstance(err, TimeoutException)+ assert "sandbox was killed or reached its end of life" in str(err)+++async def test_async_health_check_failure_returns_raw_error():+ async def check():+ raise RuntimeError("health check failed")++ original = _stream_reset()+ err = await ahandle_rpc_exception_with_health(original, check)+ assert err is original