Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

fix(sdks): raise typed, actionable errors when sandbox dies mid-request (#1419)

Mish Ushakov82add5b4ea25
diff --git a/.changeset/spotty-llamas-shout.md b/.changeset/spotty-llamas-shout.md
new file mode 100644
index 000000000..9249e70ef
--- /dev/null
+++ b/.changeset/spotty-llamas-shout.md
@@ -0,0 +1,6 @@
+---
+'e2b': patch
+'@e2b/python-sdk': patch
+---
+
+Raise a typed, actionable error when the sandbox dies while a request is in flight. When the connection is dropped mid-request (streaming RPC calls — commands, PTY, directory watch — and filesystem read/write), the SDKs now probe the sandbox health endpoint: if the sandbox is confirmed gone, a `TimeoutError` (JS) / `TimeoutException` (Python) is raised stating the sandbox was killed or reached its end of life — consistent with how requests to an already-dead sandbox surface. In all other cases the original error propagates unchanged.
diff --git a/packages/js-sdk/src/envd/api.ts b/packages/js-sdk/src/envd/api.ts
index 52529c981..9886cd0da 100644
--- a/packages/js-sdk/src/envd/api.ts
+++ b/packages/js-sdk/src/envd/api.ts
@@ -12,10 +12,12 @@ import {
formatSandboxTimeoutError,
AuthenticationError,
RateLimitError,
+ TimeoutError,
} from '../errors'
import { StartResponse, ConnectResponse } from './process/process_pb'
import { Code, ConnectError } from '@connectrpc/connect'
import { WatchDirResponse } from './filesystem/filesystem_pb'
+import { SandboxHealthCheck } from './rpc'
type ApiError = { message?: string } | string
@@ -29,6 +31,64 @@ const DEFAULT_ERROR_MAP: Record<number, (message: string) => Error> = {
507: (message) => new NotEnoughSpaceError(message),
}
+const HEALTH_CHECK_TIMEOUT_MS = 5_000
+
+/**
+ * Probes the sandbox's envd health endpoint.
+ *
+ * @param envdApi - The envd API client of the sandbox.
+ * @returns `true` if the sandbox is running, `false` if it is not, `undefined` if its state could not be determined.
+ */
+export async function checkSandboxHealth(
+ envdApi: EnvdApiClient
+): Promise<boolean | undefined> {
+ try {
+ const res = await envdApi.api.GET('/health', {
+ signal: AbortSignal.timeout(HEALTH_CHECK_TIMEOUT_MS),
+ })
+
+ if (res.response.status === 502) {
+ return false
+ }
+ if (res.response.ok) {
+ return true
+ }
+
+ return undefined
+ } catch {
+ return undefined
+ }
+}
+
+/**
+ * Handles transport-level fetch failures from envd API calls. When the connection was
+ * dropped mid-request, probes the sandbox health to tell apart the sandbox being killed
+ * from a transient network failure (e.g. a load balancer dropping the connection).
+ *
+ * @param err - The caught error, expected to be a fetch transport failure.
+ * @param checkHealth - Probe returning whether the sandbox is running, or `undefined` when unknown.
+ * @returns A `TimeoutError` when the connection was terminated mid-request and the sandbox is confirmed gone, or the original error otherwise.
+ */
+export async function handleEnvdApiFetchError(
+ err: unknown,
+ checkHealth?: SandboxHealthCheck
+): Promise<Error> {
+ // Undici surfaces a connection dropped mid-body as a TypeError with the message 'terminated'
+ if (err instanceof TypeError && err.message === 'terminated') {
+ const running = checkHealth
+ ? await checkHealth().catch(() => undefined)
+ : undefined
+
+ if (running === false) {
+ return new TimeoutError(
+ `${err.message}: The sandbox was killed or reached its end of life while the request was in flight.`
+ )
+ }
+ }
+
+ return err as Error
+}
+
/**
* Handles errors from envd API responses by mapping HTTP status codes to specific error types.
*
diff --git a/packages/js-sdk/src/envd/rpc.ts b/packages/js-sdk/src/envd/rpc.ts
index 1b559c72e..1e9aa4a3b 100644
--- a/packages/js-sdk/src/envd/rpc.ts
+++ b/packages/js-sdk/src/envd/rpc.ts
@@ -14,6 +14,25 @@ import {
} from '../errors'
import { ENVD_DEFAULT_USER } from './versions'
+/**
+ * Result of a sandbox health probe: `true` if the sandbox is running, `false` if it is not,
+ * `undefined` if its state could not be determined.
+ */
+export type SandboxHealthCheck = () => Promise<boolean | undefined>
+
+/**
+ * Checks whether the error is the signature of the connection to the sandbox being
+ * dropped mid-request — an HTTP/2 stream reset surfaced by connect as `Code.Unknown`
+ * with the message 'terminated'.
+ */
+export function isConnectionTerminatedError(err: unknown): boolean {
+ return (
+ err instanceof ConnectError &&
+ err.code === Code.Unknown &&
+ err.rawMessage === 'terminated'
+ )
+}
+
const DEFAULT_ERROR_MAP: Partial<Record<Code, (message: string) => Error>> = {
[Code.InvalidArgument]: (message) => new InvalidArgumentError(message),
[Code.Unauthenticated]: (message) => new AuthenticationError(message),
@@ -62,6 +81,36 @@ export function handleRpcError(
return err as Error
}
+/**
+ * Like {@link handleRpcError}, but when the connection to the sandbox was dropped
+ * mid-request it probes the sandbox health to tell apart the sandbox being killed
+ * from a transient network failure (e.g. a load balancer dropping the connection).
+ * When the probe confirms the sandbox is gone, a `TimeoutError` is returned —
+ * consistent with how requests to an already-dead sandbox surface.
+ *
+ * @param err - The caught error, expected to be a `ConnectError` from the gRPC transport.
+ * @param checkHealth - Probe returning whether the sandbox is running, or `undefined` when unknown.
+ * @param errorMap - Optional map of gRPC `Code` values to error factory functions that override the defaults.
+ * @returns The corresponding `Error` instance.
+ */
+export async function handleRpcErrorWithHealthCheck(
+ err: unknown,
+ checkHealth?: SandboxHealthCheck,
+ errorMap?: Partial<Record<Code, (message: string) => Error>>
+): Promise<Error> {
+ if (isConnectionTerminatedError(err) && checkHealth) {
+ const running = await checkHealth().catch(() => undefined)
+
+ if (running === false) {
+ return new TimeoutError(
+ `${(err as ConnectError).message}: The sandbox was killed or reached its end of life while the request was in flight.`
+ )
+ }
+ }
+
+ return handleRpcError(err, errorMap)
+}
+
function encode64(value: string): string {
switch (runtime) {
case 'deno':
diff --git a/packages/js-sdk/src/sandbox/commands/commandHandle.ts b/packages/js-sdk/src/sandbox/commands/commandHandle.ts
index 09713bbb8..0bf37bdb0 100644
--- a/packages/js-sdk/src/sandbox/commands/commandHandle.ts
+++ b/packages/js-sdk/src/sandbox/commands/commandHandle.ts
@@ -1,4 +1,7 @@
-import { handleRpcError } from '../../envd/rpc'
+import {
+ handleRpcErrorWithHealthCheck,
+ SandboxHealthCheck,
+} from '../../envd/rpc'
import { SandboxError } from '../../errors'
import { ConnectResponse, StartResponse } from '../../envd/process/process_pb'
import type { CommandRequestOpts } from '.'
@@ -112,7 +115,8 @@ export class CommandHandle
) => Promise<void>,
private readonly handleCloseStdin?: (
opts?: CommandRequestOpts
- ) => Promise<void>
+ ) => Promise<void>,
+ private readonly checkHealth?: SandboxHealthCheck
) {
this._wait = this.handleEvents()
}
@@ -281,7 +285,10 @@ export class CommandHandle
}
}
} catch (e) {
- this.iterationError = handleRpcError(e)
+ this.iterationError = await handleRpcErrorWithHealthCheck(
+ e,
+ this.checkHealth
+ )
} finally {
this.handleDisconnect()
}
diff --git a/packages/js-sdk/src/sandbox/commands/index.ts b/packages/js-sdk/src/sandbox/commands/index.ts
index 6302ed2d9..bbc92241f 100644
--- a/packages/js-sdk/src/sandbox/commands/index.ts
+++ b/packages/js-sdk/src/sandbox/commands/index.ts
@@ -15,12 +15,20 @@ import {
setupRequestController,
Username,
} from '../../connectionConfig'
-import { handleProcessStartEvent } from '../../envd/api'
+import {
+ checkSandboxHealth,
+ EnvdApiClient,
+ handleProcessStartEvent,
+} from '../../envd/api'
import {
Process as ProcessService,
Signal,
} from '../../envd/process/process_pb'
-import { authenticationHeader, handleRpcError } from '../../envd/rpc'
+import {
+ authenticationHeader,
+ handleRpcErrorWithHealthCheck,
+ SandboxHealthCheck,
+} from '../../envd/rpc'
import { ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE } from '../../envd/versions'
import { SandboxError } from '../../errors'
import { CommandHandle, CommandResult } from './commandHandle'
@@ -129,16 +137,16 @@ export class Commands {
private readonly defaultProcessConnectionTimeout = 60_000 // 60 seconds
private readonly envdVersion: string
+ private readonly checkHealth: SandboxHealthCheck
constructor(
transport: Transport,
- private readonly connectionConfig: ConnectionConfig,
- metadata: {
- version: string
- }
+ private readonly envdApi: EnvdApiClient,
+ private readonly connectionConfig: ConnectionConfig
) {
this.rpc = createClient(ProcessService, transport)
- this.envdVersion = metadata.version
+ this.envdVersion = envdApi.version
+ this.checkHealth = () => checkSandboxHealth(this.envdApi)
}
/**
@@ -177,7 +185,7 @@ export class Commands {
...(p.config!.cwd && { cwd: p.config!.cwd }),
}))
} catch (err) {
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -220,7 +228,7 @@ export class Commands {
}
)
} catch (err) {
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -257,7 +265,7 @@ export class Commands {
}
)
} catch (err) {
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -298,7 +306,7 @@ export class Commands {
}
}
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -354,11 +362,12 @@ export class Commands {
opts?.onStderr,
undefined,
(data, stdinOpts) => this.sendStdin(pid, data, stdinOpts),
- (stdinOpts) => this.closeStdin(pid, stdinOpts)
+ (stdinOpts) => this.closeStdin(pid, stdinOpts),
+ this.checkHealth
)
} catch (err) {
cleanup()
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -468,11 +477,12 @@ export class Commands {
opts?.onStderr,
undefined,
(data, stdinOpts) => this.sendStdin(pid, data, stdinOpts),
- (stdinOpts) => this.closeStdin(pid, stdinOpts)
+ (stdinOpts) => this.closeStdin(pid, stdinOpts),
+ this.checkHealth
)
} catch (err) {
cleanup()
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
}
diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.ts
index 089361250..283190bf0 100644
--- a/packages/js-sdk/src/sandbox/commands/pty.ts
+++ b/packages/js-sdk/src/sandbox/commands/pty.ts
@@ -19,8 +19,16 @@ import {
setupRequestController,
} from '../../connectionConfig'
import { CommandHandle } from './commandHandle'
-import { authenticationHeader, handleRpcError } from '../../envd/rpc'
-import { handleProcessStartEvent } from '../../envd/api'
+import {
+ authenticationHeader,
+ handleRpcErrorWithHealthCheck,
+ SandboxHealthCheck,
+} from '../../envd/rpc'
+import {
+ checkSandboxHealth,
+ EnvdApiClient,
+ handleProcessStartEvent,
+} from '../../envd/api'
export interface PtyCreateOpts
extends Pick<ConnectionOpts, 'requestTimeoutMs' | 'signal'> {
@@ -74,18 +82,18 @@ export type PtyConnectOpts = Pick<PtyCreateOpts, 'onData' | 'timeoutMs'> &
export class Pty {
private readonly rpc: Client<typeof ProcessService>
private readonly envdVersion: string
+ private readonly checkHealth: SandboxHealthCheck
private readonly defaultPtyConnectionTimeout = 60_000 // 60 seconds
constructor(
private readonly transport: Transport,
- private readonly connectionConfig: ConnectionConfig,
- metadata: {
- version: string
- }
+ private readonly envdApi: EnvdApiClient,
+ private readonly connectionConfig: ConnectionConfig
) {
this.rpc = createClient(ProcessService, this.transport)
- this.envdVersion = metadata.version
+ this.envdVersion = envdApi.version
+ this.checkHealth = () => checkSandboxHealth(this.envdApi)
}
/**
@@ -144,11 +152,14 @@ export class Pty {
events,
undefined,
undefined,
- opts.onData
+ opts.onData,
+ undefined,
+ undefined,
+ this.checkHealth
)
} catch (err) {
cleanup()
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -198,11 +209,14 @@ export class Pty {
events,
undefined,
undefined,
- opts?.onData
+ opts?.onData,
+ undefined,
+ undefined,
+ this.checkHealth
)
} catch (err) {
cleanup()
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -242,7 +256,7 @@ export class Pty {
}
)
} catch (err) {
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -283,7 +297,7 @@ export class Pty {
}
)
} catch (err) {
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
@@ -327,7 +341,7 @@ export class Pty {
}
}
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
}
diff --git a/packages/js-sdk/src/sandbox/filesystem/index.ts b/packages/js-sdk/src/sandbox/filesystem/index.ts
index e63e9ce3b..116361ff0 100644
--- a/packages/js-sdk/src/sandbox/filesystem/index.ts
+++ b/packages/js-sdk/src/sandbox/filesystem/index.ts
@@ -15,8 +15,17 @@ import {
Username,
} from '../../connectionConfig'
-import { handleEnvdApiError, handleWatchDirStartEvent } from '../../envd/api'
-import { authenticationHeader, handleRpcError } from '../../envd/rpc'
+import {
+ checkSandboxHealth,
+ handleEnvdApiError,
+ handleEnvdApiFetchError,
+ handleWatchDirStartEvent,
+} from '../../envd/api'
+import {
+ authenticationHeader,
+ handleRpcErrorWithHealthCheck,
+ SandboxHealthCheck,
+} from '../../envd/rpc'
import { EnvdApiClient } from '../../envd/api'
import {
@@ -54,8 +63,15 @@ const FILESYSTEM_RPC_ERROR_MAP: Partial<
[Code.NotFound]: (message: string) => new FileNotFoundError(message),
}
-function handleFilesystemRpcError(err: unknown): Error {
- return handleRpcError(err, FILESYSTEM_RPC_ERROR_MAP)
+async function handleFilesystemRpcError(
+ err: unknown,
+ checkHealth?: SandboxHealthCheck
+): Promise<Error> {
+ return handleRpcErrorWithHealthCheck(
+ err,
+ checkHealth,
+ FILESYSTEM_RPC_ERROR_MAP
+ )
}
function handleFilesystemEnvdApiError(res: {
@@ -331,6 +347,7 @@ export class Filesystem {
private readonly defaultWatchTimeout = 60_000 // 60 seconds
private readonly defaultWatchRecursive = false
+ private readonly checkHealth: SandboxHealthCheck
constructor(
transport: Transport,
@@ -338,6 +355,7 @@ export class Filesystem {
private readonly connectionConfig: ConnectionConfig
) {
this.rpc = createClient(FilesystemService, transport)
+ this.checkHealth = () => checkSandboxHealth(this.envdApi)
}
/**
@@ -421,20 +439,24 @@ export class Filesystem {
headers['Accept-Encoding'] = 'gzip'
}
- const res = await this.envdApi.api.GET('/files', {
- params: {
- query: {
- path,
- username: user,
+ const res = await this.envdApi.api
+ .GET('/files', {
+ params: {
+ query: {
+ path,
+ username: user,
+ },
},
- },
- parseAs: format === 'bytes' ? 'arrayBuffer' : format,
- signal: this.connectionConfig.getSignal(
- opts?.requestTimeoutMs,
- opts?.signal
- ),
- headers,
- })
+ parseAs: format === 'bytes' ? 'arrayBuffer' : format,
+ signal: this.connectionConfig.getSignal(
+ opts?.requestTimeoutMs,
+ opts?.signal
+ ),
+ headers,
+ })
+ .catch(async (err) => {
+ throw await handleEnvdApiFetchError(err, this.checkHealth)
+ })
const err = await handleFilesystemEnvdApiError(res)
if (err) {
@@ -565,21 +587,25 @@ export class Filesystem {
const filePath = path ?? (file as WriteEntry).path
const body = await toUploadBody(file.data, useGzip)
- const res = await this.envdApi.api.POST('/files', {
- params: {
- query: {
- path: filePath,
- username: user,
+ const res = await this.envdApi.api
+ .POST('/files', {
+ params: {
+ query: {
+ path: filePath,
+ username: user,
+ },
},
- },
- bodySerializer: () => body,
- headers,
- signal: this.connectionConfig.getSignal(
- writeOpts?.requestTimeoutMs,
- writeOpts?.signal
- ),
- body: {},
- })
+ bodySerializer: () => body,
+ headers,
+ signal: this.connectionConfig.getSignal(
+ writeOpts?.requestTimeoutMs,
+ writeOpts?.signal
+ ),
+ body: {},
+ })
+ .catch(async (err) => {
+ throw await handleEnvdApiFetchError(err, this.checkHealth)
+ })
const err = await handleFilesystemEnvdApiError(res)
if (err) {
@@ -614,21 +640,25 @@ export class Filesystem {
)
}
- const res = await this.envdApi.api.POST('/files', {
- params: {
- query: {
- path,
- username: user,
+ const res = await this.envdApi.api
+ .POST('/files', {
+ params: {
+ query: {
+ path,
+ username: user,
+ },
},
- },
- bodySerializer: () => formData,
- headers: extraHeaders,
- signal: this.connectionConfig.getSignal(
- writeOpts?.requestTimeoutMs,
- writeOpts?.signal
- ),
- body: {},
- })
+ bodySerializer: () => formData,
+ headers: extraHeaders,
+ signal: this.connectionConfig.getSignal(
+ writeOpts?.requestTimeoutMs,
+ writeOpts?.signal
+ ),
+ body: {},
+ })
+ .catch(async (err) => {
+ throw await handleEnvdApiFetchError(err, this.checkHealth)
+ })
const err = await handleFilesystemEnvdApiError(res)
if (err) {
@@ -713,7 +743,7 @@ export class Filesystem {
return entries
} catch (err) {
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
@@ -746,7 +776,7 @@ export class Filesystem {
}
}
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
@@ -786,7 +816,7 @@ export class Filesystem {
return mapEntryInfo(entry)
} catch (err) {
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
@@ -809,7 +839,7 @@ export class Filesystem {
}
)
} catch (err) {
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
@@ -842,7 +872,7 @@ export class Filesystem {
}
}
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
@@ -878,7 +908,7 @@ export class Filesystem {
return mapEntryInfo(res.entry)
} catch (err) {
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
@@ -959,10 +989,16 @@ export class Filesystem {
await handleWatchDirStartEvent(events)
clearStartTimeout()
- return new WatchHandle(cleanup, events, onEvent, opts?.onExit)
+ return new WatchHandle(
+ cleanup,
+ events,
+ onEvent,
+ opts?.onExit,
+ this.checkHealth
+ )
} catch (err) {
cleanup()
- throw handleFilesystemRpcError(err)
+ throw await handleFilesystemRpcError(err, this.checkHealth)
}
}
}
diff --git a/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts b/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts
index e29d5703d..9d55573b6 100644
--- a/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts
+++ b/packages/js-sdk/src/sandbox/filesystem/watchHandle.ts
@@ -1,4 +1,7 @@
-import { handleRpcError } from '../../envd/rpc'
+import {
+ handleRpcErrorWithHealthCheck,
+ SandboxHealthCheck,
+} from '../../envd/rpc'
import {
EventType,
WatchDirResponse,
@@ -78,7 +81,8 @@ export class WatchHandle {
private readonly handleStop: () => void,
private readonly events: AsyncIterable<WatchDirResponse>,
private readonly onEvent?: (event: FilesystemEvent) => void | Promise<void>,
- private readonly onExit?: (err?: Error) => void | Promise<void>
+ private readonly onExit?: (err?: Error) => void | Promise<void>,
+ private readonly checkHealth?: SandboxHealthCheck
) {
this.handleEvents()
}
@@ -100,7 +104,7 @@ export class WatchHandle {
}
}
} catch (err) {
- throw handleRpcError(err)
+ throw await handleRpcErrorWithHealthCheck(err, this.checkHealth)
}
}
diff --git a/packages/js-sdk/src/sandbox/index.ts b/packages/js-sdk/src/sandbox/index.ts
index 79d18b98b..1ff75e0ed 100644
--- a/packages/js-sdk/src/sandbox/index.ts
+++ b/packages/js-sdk/src/sandbox/index.ts
@@ -219,12 +219,12 @@ export class Sandbox extends SandboxApi {
this.envdApi,
this.connectionConfig
)
- this.commands = new Commands(rpcTransport, this.connectionConfig, {
- version: opts.envdVersion,
- })
- this.pty = new Pty(rpcTransport, this.connectionConfig, {
- version: opts.envdVersion,
- })
+ this.commands = new Commands(
+ rpcTransport,
+ this.envdApi,
+ this.connectionConfig
+ )
+ this.pty = new Pty(rpcTransport, this.envdApi, this.connectionConfig)
this.git = new Git(this.commands)
}
diff --git a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts
index e6f63ed50..46b569f21 100644
--- a/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts
+++ b/packages/js-sdk/tests/envd/handleEnvdApiError.test.ts
@@ -1,5 +1,5 @@
import { assert, test, describe } from 'vitest'
-import { handleEnvdApiError } from '../../src/envd/api'
+import { handleEnvdApiError, handleEnvdApiFetchError } from '../../src/envd/api'
import {
AuthenticationError,
InvalidArgumentError,
@@ -102,3 +102,32 @@ describe('handleEnvdApiError', () => {
assert.include(err?.message, '500')
})
})
+
+describe('handleEnvdApiFetchError', () => {
+ test('returns the original error for terminated fetch without a health check', async () => {
+ const original = new TypeError('terminated')
+ const err = await handleEnvdApiFetchError(original)
+ assert.strictEqual(err, original)
+ })
+
+ test('returns a TimeoutError when the health check says the sandbox is not running', async () => {
+ const err = await handleEnvdApiFetchError(
+ new TypeError('terminated'),
+ async () => false
+ )
+ assert.instanceOf(err, TimeoutError)
+ assert.include(err.message, 'sandbox was killed or reached its end of life')
+ })
+
+ test('returns the original error when the health check says the sandbox is running', async () => {
+ const original = new TypeError('terminated')
+ const err = await handleEnvdApiFetchError(original, async () => true)
+ assert.strictEqual(err, original)
+ })
+
+ test('returns the original error for other fetch failures', async () => {
+ const original = new TypeError('fetch failed')
+ const err = await handleEnvdApiFetchError(original)
+ assert.strictEqual(err, original)
+ })
+})
diff --git a/packages/js-sdk/tests/envd/handleRpcError.test.ts b/packages/js-sdk/tests/envd/handleRpcError.test.ts
index 590ae6a72..4a3f423d8 100644
--- a/packages/js-sdk/tests/envd/handleRpcError.test.ts
+++ b/packages/js-sdk/tests/envd/handleRpcError.test.ts
@@ -1,6 +1,9 @@
import { assert, test, describe } from 'vitest'
import { Code, ConnectError } from '@connectrpc/connect'
-import { handleRpcError } from '../../src/envd/rpc'
+import {
+ handleRpcError,
+ handleRpcErrorWithHealthCheck,
+} from '../../src/envd/rpc'
import {
AuthenticationError,
InvalidArgumentError,
@@ -44,9 +47,71 @@ describe('handleRpcError', () => {
assert.instanceOf(err, SandboxError)
})
+ test('falls back to generic SandboxError for Unknown "terminated"', () => {
+ const err = handleRpcError(new ConnectError('terminated', Code.Unknown))
+ assert.instanceOf(err, SandboxError)
+ assert.include(err.message, 'terminated')
+ assert.notInclude(err.message, 'killed')
+ })
+
test('returns the original error when not a ConnectError', () => {
const original = new Error('not connect')
const err = handleRpcError(original)
assert.strictEqual(err, original)
})
})
+
+describe('handleRpcErrorWithHealthCheck', () => {
+ const terminated = () => new ConnectError('terminated', Code.Unknown)
+
+ test('returns a TimeoutError when the health check says the sandbox is not running', async () => {
+ const err = await handleRpcErrorWithHealthCheck(
+ terminated(),
+ async () => false
+ )
+ assert.instanceOf(err, TimeoutError)
+ assert.include(err.message, 'sandbox was killed or reached its end of life')
+ })
+
+ test('falls back to the generic mapping when the health check says the sandbox is running', async () => {
+ const err = await handleRpcErrorWithHealthCheck(
+ terminated(),
+ async () => true
+ )
+ assert.instanceOf(err, SandboxError)
+ assert.notInstanceOf(err, TimeoutError)
+ assert.notInclude(err.message, 'killed')
+ })
+
+ test('falls back to the generic mapping when the sandbox state is unknown', async () => {
+ const err = await handleRpcErrorWithHealthCheck(
+ terminated(),
+ async () => undefined
+ )
+ assert.instanceOf(err, SandboxError)
+ assert.notInstanceOf(err, TimeoutError)
+ assert.notInclude(err.message, 'killed')
+ })
+
+ test('falls back to the generic mapping when the health check itself fails', async () => {
+ const err = await handleRpcErrorWithHealthCheck(terminated(), async () => {
+ throw new Error('health check failed')
+ })
+ assert.instanceOf(err, SandboxError)
+ assert.notInstanceOf(err, TimeoutError)
+ assert.notInclude(err.message, 'killed')
+ })
+
+ test('does not run the health check for other errors', async () => {
+ let called = false
+ const err = await handleRpcErrorWithHealthCheck(
+ new ConnectError('missing', Code.NotFound),
+ async () => {
+ called = true
+ return false
+ }
+ )
+ assert.instanceOf(err, NotFoundError)
+ assert.isFalse(called)
+ })
+})
diff --git a/packages/js-sdk/tests/sandbox/commands/sandboxKilledDuringRun.test.ts b/packages/js-sdk/tests/sandbox/commands/sandboxKilledDuringRun.test.ts
new file mode 100644
index 000000000..511a7a38a
--- /dev/null
+++ b/packages/js-sdk/tests/sandbox/commands/sandboxKilledDuringRun.test.ts
@@ -0,0 +1,20 @@
+import { expect } from 'vitest'
+import { TimeoutError } from '../../../src/index.js'
+
+import { sandboxTest, isDebug } from '../../setup.js'
+
+sandboxTest.skipIf(isDebug)(
+ 'killing the sandbox while a command is running throws an actionable error',
+ async ({ sandbox }) => {
+ const cmd = await sandbox.commands.run('sleep 60', { background: true })
+
+ await sandbox.kill()
+
+ const err = await cmd.wait().catch((e) => e)
+ expect(err).toBeInstanceOf(TimeoutError)
+ // The health check confirms the sandbox is gone, so the error states it outright
+ expect(err.message).toContain(
+ 'sandbox was killed or reached its end of life'
+ )
+ }
+)
diff --git a/packages/python-sdk/e2b/envd/api.py b/packages/python-sdk/e2b/envd/api.py
index 7074f9e6a..811d58d59 100644
--- a/packages/python-sdk/e2b/envd/api.py
+++ b/packages/python-sdk/e2b/envd/api.py
@@ -3,6 +3,7 @@ import json
from typing import Callable, Optional
+from e2b.envd.rpc import format_terminated_exception
from e2b.exceptions import (
SandboxException,
NotFoundException,
@@ -29,6 +30,85 @@ _DEFAULT_API_ERROR_MAP: dict[int, Callable[[str], Exception]] = {
}
+HEALTH_CHECK_TIMEOUT = 5 # seconds
+
+
+def check_sandbox_health(envd_api: httpx.Client) -> Optional[bool]:
+ """Probe the sandbox's envd health endpoint.
+
+ :return: ``True`` if the sandbox is running, ``False`` if it is not, ``None`` if its state could not be determined.
+ """
+ try:
+ r = envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT)
+ if r.status_code == 502:
+ return False
+ if r.is_success:
+ return True
+ return None
+ except Exception:
+ return None
+
+
+async def acheck_sandbox_health(envd_api: httpx.AsyncClient) -> Optional[bool]:
+ """Async version of :func:`check_sandbox_health`."""
+ try:
+ r = await envd_api.get(ENVD_API_HEALTH_ROUTE, timeout=HEALTH_CHECK_TIMEOUT)
+ if r.status_code == 502:
+ return False
+ if r.is_success:
+ return True
+ return None
+ except Exception:
+ return None
+
+
+def handle_envd_api_transport_exception(
+ e: Exception,
+ sandbox_running: Optional[bool] = None,
+) -> Exception:
+ """Handle transport-level errors from envd API requests.
+
+ :param e: The caught exception, expected to be a transport-level ``httpx`` error.
+ :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request.
+ :return: A ``TimeoutException`` when the connection dropped mid-request and the sandbox is confirmed gone, or the original exception unchanged otherwise.
+ """
+ # A remote protocol error (e.g. an HTTP/2 stream reset) means the connection to the
+ # sandbox was dropped mid-request — either the sandbox died or the network failed
+ if isinstance(e, httpx.RemoteProtocolError):
+ return format_terminated_exception(e, sandbox_running)
+
+ return e
+
+
+def handle_envd_api_transport_exception_with_health(
+ e: Exception,
+ envd_api: httpx.Client,
+) -> Exception:
+ """Like :func:`handle_envd_api_transport_exception`, but when the connection to the
+ sandbox was dropped mid-request it probes the sandbox health to tell apart the sandbox
+ being killed from a transient network failure (e.g. a load balancer dropping the connection).
+ """
+ sandbox_running = (
+ check_sandbox_health(envd_api)
+ if isinstance(e, httpx.RemoteProtocolError)
+ else None
+ )
+ return handle_envd_api_transport_exception(e, sandbox_running)
+
+
+async def ahandle_envd_api_transport_exception_with_health(
+ e: Exception,
+ envd_api: httpx.AsyncClient,
+) -> Exception:
+ """Async version of :func:`handle_envd_api_transport_exception_with_health`."""
+ sandbox_running = (
+ await acheck_sandbox_health(envd_api)
+ if isinstance(e, httpx.RemoteProtocolError)
+ else None
+ )
+ return handle_envd_api_transport_exception(e, sandbox_running)
+
+
def get_message(e: httpx.Response) -> str:
try:
message = e.json().get("message", e.text)
diff --git a/packages/python-sdk/e2b/envd/rpc.py b/packages/python-sdk/e2b/envd/rpc.py
index f8d263c5e..72ace254d 100644
--- a/packages/python-sdk/e2b/envd/rpc.py
+++ b/packages/python-sdk/e2b/envd/rpc.py
@@ -1,6 +1,7 @@
import base64
-from typing import Callable, Optional
+import httpcore
+from typing import Awaitable, Callable, Optional
from packaging.version import Version
from e2b_connect.client import Code, ConnectException
@@ -33,15 +34,31 @@ _DEFAULT_RPC_ERROR_MAP: dict[Code, Callable[[str], Exception]] = {
}
+def format_terminated_exception(
+ e: Exception,
+ sandbox_running: Optional[bool],
+) -> Exception:
+ """Handle an exception for a connection to the sandbox dropped mid-request: when a
+ sandbox health probe confirmed the sandbox is gone (``sandbox_running is False``),
+ return a ``TimeoutException``; otherwise return the original error unchanged."""
+ if sandbox_running is False:
+ return TimeoutException(
+ f"{e}: The sandbox was killed or reached its end of life while the request was in flight."
+ )
+ return e
+
+
def handle_rpc_exception(
e: Exception,
error_map: Optional[dict[Code, Callable[[str], Exception]]] = None,
+ sandbox_running: Optional[bool] = None,
):
"""Handle errors from envd RPC calls by mapping gRPC status codes to specific exception types.
- :param e: The caught exception, expected to be a ``ConnectException``.
+ :param e: The caught exception, expected to be a ``ConnectException`` or a transport-level ``httpcore`` error.
:param error_map: Optional map of gRPC codes to exception factories that override the defaults.
- :return: The corresponding exception, or the original exception if not a ``ConnectException``.
+ :param sandbox_running: Result of a sandbox health probe (``None`` when unknown), used to disambiguate a connection dropped mid-request.
+ :return: The corresponding exception. A connection dropped mid-request with the sandbox confirmed gone becomes a ``TimeoutException``; non-``ConnectException`` errors are otherwise returned as-is.
"""
if isinstance(e, ConnectException):
if error_map and e.status in error_map:
@@ -51,8 +68,46 @@ def handle_rpc_exception(
return _DEFAULT_RPC_ERROR_MAP[e.status](e.message)
return SandboxException(f"{e.status}: {e.message}")
- else:
- return e
+
+ # A remote protocol error (e.g. an HTTP/2 stream reset) means the connection to the
+ # sandbox was dropped mid-request — either the sandbox died or the network failed
+ if isinstance(e, httpcore.RemoteProtocolError):
+ return format_terminated_exception(e, sandbox_running)
+
+ return e
+
+
+def handle_rpc_exception_with_health(
+ e: Exception,
+ check_health: Optional[Callable[[], Optional[bool]]] = None,
+ error_map: Optional[dict[Code, Callable[[str], Exception]]] = None,
+):
+ """Like :func:`handle_rpc_exception`, but when the connection to the sandbox was
+ dropped mid-request it probes the sandbox health to tell apart the sandbox being
+ killed from a transient network failure (e.g. a load balancer dropping the connection).
+ """
+ sandbox_running = None
+ if check_health is not None and isinstance(e, httpcore.RemoteProtocolError):
+ try:
+ sandbox_running = check_health()
+ except Exception:
+ sandbox_running = None
+ return handle_rpc_exception(e, error_map, sandbox_running)
+
+
+async def ahandle_rpc_exception_with_health(
+ e: Exception,
+ check_health: Optional[Callable[[], Awaitable[Optional[bool]]]] = None,
+ error_map: Optional[dict[Code, Callable[[str], Exception]]] = None,
+):
+ """Async version of :func:`handle_rpc_exception_with_health`."""
+ sandbox_running = None
+ if check_health is not None and isinstance(e, httpcore.RemoteProtocolError):
+ try:
+ sandbox_running = await check_health()
+ except Exception:
+ sandbox_running = None
+ return handle_rpc_exception(e, error_map, sandbox_running)
def authentication_header(
diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.py
index 1280d0fc8..8142d5321 100644
--- a/packages/python-sdk/e2b/sandbox_async/commands/command.py
+++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py
@@ -2,6 +2,7 @@ from typing import Dict, List, Literal, Optional, Union, overload
import e2b_connect
import httpcore
+import httpx
from packaging.version import Version
from e2b.connection_config import (
ConnectionConfig,
@@ -10,7 +11,8 @@ from e2b.connection_config import (
KEEPALIVE_PING_INTERVAL_SEC,
)
from e2b.envd.process import process_connect, process_pb2
-from e2b.envd.rpc import authentication_header, handle_rpc_exception
+from e2b.envd.api import acheck_sandbox_health
+from e2b.envd.rpc import authentication_header, ahandle_rpc_exception_with_health
from e2b.envd.versions import ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE
from e2b.exceptions import SandboxException
from e2b.sandbox.commands.main import ProcessInfo
@@ -30,9 +32,11 @@ class Commands:
connection_config: ConnectionConfig,
pool: httpcore.AsyncConnectionPool,
envd_version: Version,
+ envd_api: httpx.AsyncClient,
) -> None:
self._connection_config = connection_config
self._envd_version = envd_version
+ self._check_health = lambda: acheck_sandbox_health(envd_api)
self._rpc = process_connect.ProcessClient(
envd_api_url,
# TODO: Fix and enable compression again — the headers compression is not solved for streaming.
@@ -72,7 +76,7 @@ class Commands:
for p in res.processes
]
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def kill(
self,
@@ -103,7 +107,7 @@ class Commands:
if isinstance(e, e2b_connect.ConnectException):
if e.status == e2b_connect.Code.not_found:
return False
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def send_stdin(
self,
@@ -131,7 +135,7 @@ class Commands:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def close_stdin(
self,
@@ -162,7 +166,7 @@ class Commands:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
@overload
async def run(
@@ -318,9 +322,10 @@ class Commands:
handle_close_stdin=lambda request_timeout=None: self.close_stdin(
pid, request_timeout
),
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def connect(
self,
@@ -376,6 +381,7 @@ class Commands:
handle_close_stdin=lambda request_timeout=None: self.close_stdin(
pid, request_timeout
),
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py
index e74ab6ce7..7b094cd6f 100644
--- a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py
+++ b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py
@@ -5,12 +5,13 @@ from typing import (
Callable,
Any,
AsyncGenerator,
+ Awaitable,
Union,
Tuple,
Coroutine,
)
-from e2b.envd.rpc import handle_rpc_exception
+from e2b.envd.rpc import ahandle_rpc_exception_with_health
from e2b.envd.process import process_pb2
from e2b.exceptions import SandboxException
from e2b.sandbox.commands.command_handle import (
@@ -89,11 +90,13 @@ class AsyncCommandHandle:
handle_close_stdin: Optional[
Callable[[Optional[float]], Coroutine[Any, Any, None]]
] = None,
+ check_health: Optional[Callable[[], Awaitable[Optional[bool]]]] = None,
):
self._pid = pid
self._handle_kill = handle_kill
self._handle_send_stdin = handle_send_stdin
self._handle_close_stdin = handle_close_stdin
+ self._check_health = check_health
self._events = events
self._stdout: str = ""
@@ -170,7 +173,9 @@ class AsyncCommandHandle:
except StopAsyncIteration:
pass
except Exception as e:
- self._iteration_exception = handle_rpc_exception(e)
+ self._iteration_exception = await ahandle_rpc_exception_with_health(
+ e, self._check_health
+ )
async def wait(self) -> CommandResult:
"""
diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.py
index 5b7b69aef..809f061b8 100644
--- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py
+++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py
@@ -2,6 +2,7 @@ from typing import Dict, Optional
import e2b_connect
import httpcore
+import httpx
from packaging.version import Version
from e2b.envd.process import process_connect, process_pb2
@@ -12,7 +13,8 @@ from e2b.connection_config import (
KEEPALIVE_PING_INTERVAL_SEC,
)
from e2b.exceptions import SandboxException
-from e2b.envd.rpc import authentication_header, handle_rpc_exception
+from e2b.envd.api import acheck_sandbox_health
+from e2b.envd.rpc import authentication_header, ahandle_rpc_exception_with_health
from e2b.sandbox.commands.command_handle import PtySize
from e2b.sandbox_async.commands.command_handle import (
AsyncCommandHandle,
@@ -32,9 +34,11 @@ class Pty:
connection_config: ConnectionConfig,
pool: httpcore.AsyncConnectionPool,
envd_version: Version,
+ envd_api: httpx.AsyncClient,
) -> None:
self._connection_config = connection_config
self._envd_version = envd_version
+ self._check_health = lambda: acheck_sandbox_health(envd_api)
self._rpc = process_connect.ProcessClient(
envd_api_url,
# TODO: Fix and enable compression again — the headers compression is not solved for streaming.
@@ -72,7 +76,7 @@ class Pty:
if isinstance(e, e2b_connect.ConnectException):
if e.status == e2b_connect.Code.not_found:
return False
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def send_stdin(
self,
@@ -100,7 +104,7 @@ class Pty:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def create(
self,
@@ -164,9 +168,10 @@ class Pty:
handle_kill=lambda: self.kill(start_event.event.start.pid),
events=events,
on_pty=on_data,
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def connect(
self,
@@ -211,9 +216,10 @@ class Pty:
handle_kill=lambda: self.kill(start_event.event.start.pid),
events=events,
on_pty=on_data,
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def resize(
self,
diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
index 6c88811b0..208a08c0f 100644
--- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
@@ -14,9 +14,14 @@ from e2b.connection_config import (
Username,
default_username,
)
-from e2b.envd.api import ENVD_API_FILES_ROUTE, ahandle_envd_api_exception
+from e2b.envd.api import (
+ ENVD_API_FILES_ROUTE,
+ acheck_sandbox_health,
+ ahandle_envd_api_exception,
+ ahandle_envd_api_transport_exception_with_health,
+)
from e2b.envd.filesystem import filesystem_connect, filesystem_pb2
-from e2b.envd.rpc import authentication_header, handle_rpc_exception
+from e2b.envd.rpc import authentication_header, ahandle_rpc_exception_with_health
from e2b.envd.versions import (
ENVD_DEFAULT_USER,
ENVD_FILE_METADATA,
@@ -56,8 +61,12 @@ _FILESYSTEM_HTTP_ERROR_MAP = {
}
-def _handle_filesystem_rpc_exception(e: Exception) -> Exception:
- return handle_rpc_exception(e, _FILESYSTEM_RPC_ERROR_MAP)
+async def _ahandle_filesystem_rpc_exception(
+ e: Exception, envd_api: httpx.AsyncClient
+) -> Exception:
+ return await ahandle_rpc_exception_with_health(
+ e, lambda: acheck_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP
+ )
async def _ahandle_filesystem_envd_api_exception(r):
@@ -178,12 +187,17 @@ class Filesystem:
if gzip:
headers["Accept-Encoding"] = "gzip"
- r = await self._envd_api.get(
- ENVD_API_FILES_ROUTE,
- params=params,
- headers=headers,
- timeout=self._connection_config.get_request_timeout(request_timeout),
- )
+ try:
+ r = await self._envd_api.get(
+ ENVD_API_FILES_ROUTE,
+ params=params,
+ headers=headers,
+ timeout=self._connection_config.get_request_timeout(request_timeout),
+ )
+ except httpx.RemoteProtocolError as e:
+ raise await ahandle_envd_api_transport_exception_with_health(
+ e, self._envd_api
+ )
err = await _ahandle_filesystem_envd_api_exception(r)
if err:
@@ -295,15 +309,20 @@ class Filesystem:
if gzip:
headers["Content-Encoding"] = "gzip"
- r = await self._envd_api.post(
- ENVD_API_FILES_ROUTE,
- content=to_upload_body(file_data, gzip),
- headers=headers,
- params=params,
- timeout=self._connection_config.get_request_timeout(
- request_timeout
- ),
- )
+ try:
+ r = await self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ content=to_upload_body(file_data, gzip),
+ headers=headers,
+ params=params,
+ timeout=self._connection_config.get_request_timeout(
+ request_timeout
+ ),
+ )
+ except httpx.RemoteProtocolError as e:
+ raise await ahandle_envd_api_transport_exception_with_health(
+ e, self._envd_api
+ )
err = await _ahandle_filesystem_envd_api_exception(r)
if err:
@@ -335,13 +354,20 @@ class Filesystem:
if len(httpx_files) == 0:
return []
- r = await self._envd_api.post(
- ENVD_API_FILES_ROUTE,
- files=httpx_files,
- params=params,
- headers=extra_headers,
- timeout=self._connection_config.get_request_timeout(request_timeout),
- )
+ try:
+ r = await self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ files=httpx_files,
+ params=params,
+ headers=extra_headers,
+ timeout=self._connection_config.get_request_timeout(
+ request_timeout
+ ),
+ )
+ except httpx.RemoteProtocolError as e:
+ raise await ahandle_envd_api_transport_exception_with_health(
+ e, self._envd_api
+ )
err = await _ahandle_filesystem_envd_api_exception(r)
if err:
@@ -395,7 +421,7 @@ class Filesystem:
return entries
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
async def exists(
self,
@@ -427,7 +453,7 @@ class Filesystem:
if isinstance(e, connect.ConnectException):
if e.status == connect.Code.not_found:
return False
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
async def get_info(
self,
@@ -455,7 +481,7 @@ class Filesystem:
return map_entry_info(r.entry)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
async def remove(
self,
@@ -479,7 +505,7 @@ class Filesystem:
headers=authentication_header(self._envd_version, user),
)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
async def rename(
self,
@@ -512,7 +538,7 @@ class Filesystem:
return map_entry_info(r.entry)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
async def make_dir(
self,
@@ -543,7 +569,7 @@ class Filesystem:
if isinstance(e, connect.ConnectException):
if e.status == connect.Code.already_exists:
return False
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
async def watch_dir(
self,
@@ -615,6 +641,11 @@ class Filesystem:
f"Failed to start watch: expected start event, got {start_event}",
)
- return AsyncWatchHandle(events=events, on_event=on_event, on_exit=on_exit)
+ return AsyncWatchHandle(
+ events=events,
+ on_event=on_event,
+ on_exit=on_exit,
+ check_health=lambda: acheck_sandbox_health(self._envd_api),
+ )
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py b/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py
index a33d13fd1..092b066f3 100644
--- a/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py
+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/watch_handle.py
@@ -1,9 +1,9 @@
import asyncio
import inspect
-from typing import Any, AsyncGenerator, Optional
+from typing import Any, AsyncGenerator, Awaitable, Callable, Optional
-from e2b.envd.rpc import handle_rpc_exception
+from e2b.envd.rpc import ahandle_rpc_exception_with_health
from e2b.envd.filesystem.filesystem_pb2 import WatchDirResponse
from e2b.sandbox.filesystem.filesystem import map_entry_info
from e2b.sandbox.filesystem.watch_handle import FilesystemEvent, map_event_type
@@ -22,10 +22,12 @@ class AsyncWatchHandle:
events: AsyncGenerator[WatchDirResponse, Any],
on_event: OutputHandler[FilesystemEvent],
on_exit: Optional[OutputHandler[Exception]] = None,
+ check_health: Optional[Callable[[], Awaitable[Optional[bool]]]] = None,
):
self._events = events
self._on_event = on_event
self._on_exit = on_exit
+ self._check_health = check_health
self._wait = asyncio.create_task(self._handle_events())
@@ -56,7 +58,7 @@ class AsyncWatchHandle:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def _handle_events(self):
try:
diff --git a/packages/python-sdk/e2b/sandbox_async/main.py b/packages/python-sdk/e2b/sandbox_async/main.py
index a09ea3336..a4c5d9b7d 100644
--- a/packages/python-sdk/e2b/sandbox_async/main.py
+++ b/packages/python-sdk/e2b/sandbox_async/main.py
@@ -122,12 +122,14 @@ class AsyncSandbox(SandboxApi):
self.connection_config,
self._transport.pool,
self._envd_version,
+ self._envd_api,
)
self._pty = Pty(
self.envd_api_url,
self.connection_config,
self._transport.pool,
self._envd_version,
+ self._envd_api,
)
self._git = Git(self._commands)
diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.py
index e2252bf4a..0f0585ad8 100644
--- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py
+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py
@@ -2,6 +2,7 @@ import threading
from typing import Callable, Dict, List, Literal, Optional, Union, overload
import e2b_connect
+import httpx
from packaging.version import Version
from e2b.api.client_sync import get_envd_transport
from e2b.connection_config import (
@@ -11,7 +12,8 @@ from e2b.connection_config import (
KEEPALIVE_PING_INTERVAL_SEC,
)
from e2b.envd.process import process_connect, process_pb2
-from e2b.envd.rpc import authentication_header, handle_rpc_exception
+from e2b.envd.api import check_sandbox_health
+from e2b.envd.rpc import authentication_header, handle_rpc_exception_with_health
from e2b.envd.versions import ENVD_COMMANDS_STDIN, ENVD_ENVD_CLOSE
from e2b.exceptions import SandboxException
from e2b.sandbox.commands.main import ProcessInfo
@@ -35,6 +37,14 @@ class Commands:
self._envd_version = envd_version
self._thread_local = threading.local()
+ def _create_envd_api(self) -> httpx.Client:
+ transport = get_envd_transport(self._connection_config)
+ return httpx.Client(
+ base_url=self._envd_api_url,
+ transport=transport,
+ headers=self._connection_config.sandbox_headers,
+ )
+
def _create_rpc(self) -> process_connect.ProcessClient:
transport = get_envd_transport(self._connection_config)
return process_connect.ProcessClient(
@@ -46,6 +56,14 @@ class Commands:
headers=self._connection_config.sandbox_headers,
)
+ @property
+ def _envd_api(self) -> httpx.Client:
+ envd_api = getattr(self._thread_local, "envd_api", None)
+ if envd_api is None:
+ envd_api = self._create_envd_api()
+ self._thread_local.envd_api = envd_api
+ return envd_api
+
@property
def _rpc(self) -> process_connect.ProcessClient:
rpc = getattr(self._thread_local, "rpc", None)
@@ -54,6 +72,9 @@ class Commands:
self._thread_local.rpc = rpc
return rpc
+ def _check_health(self) -> Optional[bool]:
+ return check_sandbox_health(self._envd_api)
+
def list(
self,
request_timeout: Optional[float] = None,
@@ -84,7 +105,7 @@ class Commands:
for p in res.processes
]
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def kill(
self,
@@ -115,7 +136,7 @@ class Commands:
if isinstance(e, e2b_connect.ConnectException):
if e.status == e2b_connect.Code.not_found:
return False
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def send_stdin(
self,
@@ -143,7 +164,7 @@ class Commands:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def close_stdin(
self,
@@ -174,7 +195,7 @@ class Commands:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
@overload
def run(
@@ -329,9 +350,10 @@ class Commands:
handle_close_stdin=lambda request_timeout=None: self.close_stdin(
pid, request_timeout
),
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def connect(
self,
@@ -381,6 +403,7 @@ class Commands:
handle_close_stdin=lambda request_timeout=None: self.close_stdin(
pid, request_timeout
),
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py
index f9b4f6af0..27138b9d8 100644
--- a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py
+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py
@@ -1,6 +1,6 @@
from typing import Optional, Callable, Any, Generator, Union, Tuple
-from e2b.envd.rpc import handle_rpc_exception
+from e2b.envd.rpc import handle_rpc_exception_with_health
from e2b.envd.process import process_pb2
from e2b.exceptions import SandboxException
from e2b.sandbox.commands.command_handle import (
@@ -37,11 +37,13 @@ class CommandHandle:
Callable[[Union[str, bytes], Optional[float]], None]
] = None,
handle_close_stdin: Optional[Callable[[Optional[float]], None]] = None,
+ check_health: Optional[Callable[[], Optional[bool]]] = None,
):
self._pid = pid
self._handle_kill = handle_kill
self._handle_send_stdin = handle_send_stdin
self._handle_close_stdin = handle_close_stdin
+ self._check_health = check_health
self._events = events
self._stdout: str = ""
@@ -90,7 +92,7 @@ class CommandHandle:
error=event.event.end.error,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def disconnect(self) -> None:
"""
@@ -128,7 +130,9 @@ class CommandHandle:
except StopIteration:
pass
except Exception as e:
- self._iteration_exception = handle_rpc_exception(e)
+ self._iteration_exception = handle_rpc_exception_with_health(
+ e, self._check_health
+ )
if self._iteration_exception:
raise self._iteration_exception
diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py
index 34fc7bfcc..2abf0ed85 100644
--- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py
+++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py
@@ -1,4 +1,5 @@
import e2b_connect
+import httpx
import threading
from typing import Dict, Optional
@@ -13,7 +14,8 @@ from e2b.connection_config import (
KEEPALIVE_PING_INTERVAL_SEC,
)
from e2b.exceptions import SandboxException
-from e2b.envd.rpc import authentication_header, handle_rpc_exception
+from e2b.envd.api import check_sandbox_health
+from e2b.envd.rpc import authentication_header, handle_rpc_exception_with_health
from e2b.sandbox.commands.command_handle import PtySize
from e2b.sandbox_sync.commands.command_handle import CommandHandle
@@ -34,6 +36,14 @@ class Pty:
self._envd_version = envd_version
self._thread_local = threading.local()
+ def _create_envd_api(self) -> httpx.Client:
+ transport = get_envd_transport(self._connection_config)
+ return httpx.Client(
+ base_url=self._envd_api_url,
+ transport=transport,
+ headers=self._connection_config.sandbox_headers,
+ )
+
def _create_rpc(self) -> process_connect.ProcessClient:
transport = get_envd_transport(self._connection_config)
return process_connect.ProcessClient(
@@ -45,6 +55,14 @@ class Pty:
headers=self._connection_config.sandbox_headers,
)
+ @property
+ def _envd_api(self) -> httpx.Client:
+ envd_api = getattr(self._thread_local, "envd_api", None)
+ if envd_api is None:
+ envd_api = self._create_envd_api()
+ self._thread_local.envd_api = envd_api
+ return envd_api
+
@property
def _rpc(self) -> process_connect.ProcessClient:
rpc = getattr(self._thread_local, "rpc", None)
@@ -53,6 +71,9 @@ class Pty:
self._thread_local.rpc = rpc
return rpc
+ def _check_health(self) -> Optional[bool]:
+ return check_sandbox_health(self._envd_api)
+
def kill(
self,
pid: int,
@@ -81,7 +102,7 @@ class Pty:
if isinstance(e, e2b_connect.ConnectException):
if e.status == e2b_connect.Code.not_found:
return False
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def send_stdin(
self,
@@ -109,7 +130,7 @@ class Pty:
),
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def create(
self,
@@ -170,9 +191,10 @@ class Pty:
pid=start_event.event.start.pid,
handle_kill=lambda: self.kill(start_event.event.start.pid),
events=events,
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def connect(
self,
@@ -214,9 +236,10 @@ class Pty:
pid=start_event.event.start.pid,
handle_kill=lambda: self.kill(start_event.event.start.pid),
events=events,
+ check_health=self._check_health,
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
def resize(
self,
diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py
index 7a233c1f8..d43767bf5 100644
--- a/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py
+++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/filesystem.py
@@ -15,9 +15,14 @@ from e2b.connection_config import (
)
from e2b_connect.client import Code
-from e2b.envd.api import ENVD_API_FILES_ROUTE, handle_envd_api_exception
+from e2b.envd.api import (
+ ENVD_API_FILES_ROUTE,
+ check_sandbox_health,
+ handle_envd_api_exception,
+ handle_envd_api_transport_exception_with_health,
+)
from e2b.envd.filesystem import filesystem_connect, filesystem_pb2
-from e2b.envd.rpc import authentication_header, handle_rpc_exception
+from e2b.envd.rpc import authentication_header, handle_rpc_exception_with_health
from e2b.envd.versions import (
ENVD_DEFAULT_USER,
ENVD_FILE_METADATA,
@@ -55,8 +60,10 @@ _FILESYSTEM_HTTP_ERROR_MAP = {
}
-def _handle_filesystem_rpc_exception(e: Exception) -> Exception:
- return handle_rpc_exception(e, _FILESYSTEM_RPC_ERROR_MAP)
+def _handle_filesystem_rpc_exception(e: Exception, envd_api: httpx.Client) -> Exception:
+ return handle_rpc_exception_with_health(
+ e, lambda: check_sandbox_health(envd_api), _FILESYSTEM_RPC_ERROR_MAP
+ )
def _handle_filesystem_envd_api_exception(r):
@@ -200,12 +207,15 @@ class Filesystem:
if gzip:
headers["Accept-Encoding"] = "gzip"
- r = self._envd_api.get(
- ENVD_API_FILES_ROUTE,
- params=params,
- headers=headers,
- timeout=self._connection_config.get_request_timeout(request_timeout),
- )
+ try:
+ r = self._envd_api.get(
+ ENVD_API_FILES_ROUTE,
+ params=params,
+ headers=headers,
+ timeout=self._connection_config.get_request_timeout(request_timeout),
+ )
+ except httpx.RemoteProtocolError as e:
+ raise handle_envd_api_transport_exception_with_health(e, self._envd_api)
err = _handle_filesystem_envd_api_exception(r)
if err:
@@ -316,15 +326,20 @@ class Filesystem:
if gzip:
headers["Content-Encoding"] = "gzip"
- r = self._envd_api.post(
- ENVD_API_FILES_ROUTE,
- content=to_upload_body(file_data, gzip),
- headers=headers,
- params=params,
- timeout=self._connection_config.get_request_timeout(
- request_timeout
- ),
- )
+ try:
+ r = self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ content=to_upload_body(file_data, gzip),
+ headers=headers,
+ params=params,
+ timeout=self._connection_config.get_request_timeout(
+ request_timeout
+ ),
+ )
+ except httpx.RemoteProtocolError as e:
+ raise handle_envd_api_transport_exception_with_health(
+ e, self._envd_api
+ )
err = _handle_filesystem_envd_api_exception(r)
if err:
@@ -350,13 +365,18 @@ class Filesystem:
if len(httpx_files) == 0:
return []
- r = self._envd_api.post(
- ENVD_API_FILES_ROUTE,
- files=httpx_files,
- params=params,
- headers=extra_headers,
- timeout=self._connection_config.get_request_timeout(request_timeout),
- )
+ try:
+ r = self._envd_api.post(
+ ENVD_API_FILES_ROUTE,
+ files=httpx_files,
+ params=params,
+ headers=extra_headers,
+ timeout=self._connection_config.get_request_timeout(
+ request_timeout
+ ),
+ )
+ except httpx.RemoteProtocolError as e:
+ raise handle_envd_api_transport_exception_with_health(e, self._envd_api)
err = _handle_filesystem_envd_api_exception(r)
if err:
@@ -410,7 +430,7 @@ class Filesystem:
return entries
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
def exists(
self,
@@ -441,7 +461,7 @@ class Filesystem:
if isinstance(e, e2b_connect.ConnectException):
if e.status == e2b_connect.Code.not_found:
return False
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
def get_info(
self,
@@ -469,7 +489,7 @@ class Filesystem:
return map_entry_info(r.entry)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
def remove(
self,
@@ -493,7 +513,7 @@ class Filesystem:
headers=authentication_header(self._envd_version, user),
)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
def rename(
self,
@@ -526,7 +546,7 @@ class Filesystem:
return map_entry_info(r.entry)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
def make_dir(
self,
@@ -557,7 +577,7 @@ class Filesystem:
if isinstance(e, e2b_connect.ConnectException):
if e.status == e2b_connect.Code.already_exists:
return False
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
def watch_dir(
self,
@@ -615,6 +635,10 @@ class Filesystem:
},
)
except Exception as e:
- raise _handle_filesystem_rpc_exception(e)
+ raise _handle_filesystem_rpc_exception(e, self._envd_api)
- return WatchHandle(lambda: self._rpc, r.watcher_id)
+ return WatchHandle(
+ lambda: self._rpc,
+ r.watcher_id,
+ lambda: check_sandbox_health(self._envd_api),
+ )
diff --git a/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py b/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py
index 4b47b0072..84370e013 100644
--- a/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py
+++ b/packages/python-sdk/e2b/sandbox_sync/filesystem/watch_handle.py
@@ -1,4 +1,4 @@
-from typing import Callable, List
+from typing import Callable, List, Optional
from e2b import SandboxException
from e2b.envd.filesystem import filesystem_connect
@@ -6,7 +6,7 @@ from e2b.envd.filesystem.filesystem_pb2 import (
GetWatcherEventsRequest,
RemoveWatcherRequest,
)
-from e2b.envd.rpc import handle_rpc_exception
+from e2b.envd.rpc import handle_rpc_exception_with_health
from e2b.sandbox.filesystem.filesystem import map_entry_info
from e2b.sandbox.filesystem.watch_handle import FilesystemEvent, map_event_type
@@ -23,9 +23,11 @@ class WatchHandle:
self,
get_rpc: Callable[[], filesystem_connect.FilesystemClient],
watcher_id: str,
+ check_health: Optional[Callable[[], Optional[bool]]] = None,
):
self._get_rpc = get_rpc
self._watcher_id = watcher_id
+ self._check_health = check_health
self._closed = False
def stop(self):
@@ -38,7 +40,7 @@ class WatchHandle:
RemoveWatcherRequest(watcher_id=self._watcher_id)
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
self._closed = True
@@ -56,7 +58,7 @@ class WatchHandle:
GetWatcherEventsRequest(watcher_id=self._watcher_id)
)
except Exception as e:
- raise handle_rpc_exception(e)
+ raise handle_rpc_exception_with_health(e, self._check_health)
events = []
for event in r.events:
diff --git a/packages/python-sdk/e2b_connect/client.py b/packages/python-sdk/e2b_connect/client.py
index 7e1ced394..e32fa7de5 100644
--- a/packages/python-sdk/e2b_connect/client.py
+++ b/packages/python-sdk/e2b_connect/client.py
@@ -361,6 +361,8 @@ class Client:
},
}
+ # Note: no retry here — generator functions don't execute until iterated, so a
+ # call-level retry never fires, and retrying mid-stream would replay delivered events.
async def acall_server_stream(
self,
req,
diff --git a/packages/python-sdk/tests/async/sandbox_async/commands/test_sandbox_killed_during_run.py b/packages/python-sdk/tests/async/sandbox_async/commands/test_sandbox_killed_during_run.py
new file mode 100644
index 000000000..2f83f210b
--- /dev/null
+++ b/packages/python-sdk/tests/async/sandbox_async/commands/test_sandbox_killed_during_run.py
@@ -0,0 +1,16 @@
+import pytest
+
+from e2b import AsyncSandbox, TimeoutException
+
+
+async def test_kill_sandbox_while_command_is_running(async_sandbox: AsyncSandbox):
+ cmd = await async_sandbox.commands.run("sleep 60", background=True)
+
+ await async_sandbox.kill()
+
+ with pytest.raises(TimeoutException) as exc_info:
+ await cmd.wait()
+
+ # The health check confirms the sandbox is gone, so the error states it outright
+ assert "sandbox was killed or reached its end of life" in str(exc_info.value)
diff --git a/packages/python-sdk/tests/sync/sandbox_sync/commands/test_sandbox_killed_during_run.py b/packages/python-sdk/tests/sync/sandbox_sync/commands/test_sandbox_killed_during_run.py
new file mode 100644
index 000000000..81dd1cb28
--- /dev/null
+++ b/packages/python-sdk/tests/sync/sandbox_sync/commands/test_sandbox_killed_during_run.py
@@ -0,0 +1,16 @@
+import pytest
+
+from e2b import Sandbox, TimeoutException
+
+
+def test_kill_sandbox_while_command_is_running(sandbox: Sandbox):
+ cmd = sandbox.commands.run("sleep 60", background=True)
+
+ sandbox.kill()
+
+ with pytest.raises(TimeoutException) as exc_info:
+ cmd.wait()
+
+ # The health check confirms the sandbox is gone, so the error states it outright
+ assert "sandbox was killed or reached its end of life" in str(exc_info.value)
diff --git a/packages/python-sdk/tests/test_envd_api_exception.py b/packages/python-sdk/tests/test_envd_api_exception.py
index 5c501391c..7c0694da7 100644
--- a/packages/python-sdk/tests/test_envd_api_exception.py
+++ b/packages/python-sdk/tests/test_envd_api_exception.py
@@ -1,4 +1,9 @@
-from e2b.envd.api import format_envd_api_exception
+import httpx
+
+from e2b.envd.api import (
+ format_envd_api_exception,
+ handle_envd_api_transport_exception,
+)
from e2b.exceptions import (
AuthenticationException,
InvalidArgumentException,
@@ -45,3 +50,35 @@ def test_falls_back_to_sandbox_exception():
err = format_envd_api_exception(500, "Internal error")
assert isinstance(err, SandboxException)
assert "500" in str(err)
+
+
+def test_returns_raw_remote_protocol_error_without_health_result():
+ original = httpx.RemoteProtocolError("peer closed connection")
+ err = handle_envd_api_transport_exception(original)
+ assert err is original
+
+
+def test_returns_raw_network_errors():
+ original = httpx.ReadError("read failed")
+ err = handle_envd_api_transport_exception(original)
+ assert err is original
+
+
+def test_returns_original_when_not_transport_error():
+ original = ValueError("not transport")
+ err = handle_envd_api_transport_exception(original)
+ assert err is original
+
+
+def test_health_result_confirms_sandbox_killed():
+ err = handle_envd_api_transport_exception(
+ httpx.RemoteProtocolError("peer closed connection"), sandbox_running=False
+ )
+ assert isinstance(err, TimeoutException)
+ assert "sandbox was killed or reached its end of life" in str(err)
+
+
+def test_health_result_running_returns_raw_error():
+ original = httpx.RemoteProtocolError("peer closed connection")
+ err = handle_envd_api_transport_exception(original, sandbox_running=True)
+ assert err is original
diff --git a/packages/python-sdk/tests/test_envd_rpc_exception.py b/packages/python-sdk/tests/test_envd_rpc_exception.py
index 42a455ee2..71109d3da 100644
--- a/packages/python-sdk/tests/test_envd_rpc_exception.py
+++ b/packages/python-sdk/tests/test_envd_rpc_exception.py
@@ -1,5 +1,11 @@
+import httpcore
+
from e2b_connect.client import Code, ConnectException
-from e2b.envd.rpc import handle_rpc_exception
+from e2b.envd.rpc import (
+ ahandle_rpc_exception_with_health,
+ handle_rpc_exception,
+ handle_rpc_exception_with_health,
+)
from e2b.exceptions import (
AuthenticationException,
InvalidArgumentException,
@@ -41,7 +47,82 @@ def test_falls_back_to_sandbox_exception():
assert isinstance(err, SandboxException)
+def test_returns_raw_remote_protocol_error_without_health_result():
+ original = httpcore.RemoteProtocolError(
+ "<StreamReset stream_id:1, error_code:2, remote_reset:True>"
+ )
+ err = handle_rpc_exception(original)
+ assert err is original
+
+
+def test_returns_raw_network_errors():
+ original = httpcore.ReadError("read failed")
+ err = handle_rpc_exception(original)
+ assert err is original
+
+
def test_returns_original_when_not_connect_exception():
original = ValueError("not connect")
err = handle_rpc_exception(original)
assert err is original
+
+
+def _stream_reset() -> httpcore.RemoteProtocolError:
+ return httpcore.RemoteProtocolError(
+ "<StreamReset stream_id:1, error_code:2, remote_reset:True>"
+ )
+
+
+def test_health_check_confirms_sandbox_killed():
+ err = handle_rpc_exception_with_health(_stream_reset(), lambda: False)
+ assert isinstance(err, TimeoutException)
+ assert "sandbox was killed or reached its end of life" in str(err)
+
+
+def test_health_check_running_returns_raw_error():
+ original = _stream_reset()
+ err = handle_rpc_exception_with_health(original, lambda: True)
+ assert err is original
+
+
+def test_health_check_unknown_returns_raw_error():
+ original = _stream_reset()
+ err = handle_rpc_exception_with_health(original, lambda: None)
+ assert err is original
+
+
+def test_health_check_failure_returns_raw_error():
+ def check():
+ raise RuntimeError("health check failed")
+
+ original = _stream_reset()
+ err = handle_rpc_exception_with_health(original, check)
+ assert err is original
+
+
+def test_health_check_not_run_for_other_exceptions():
+ def fail():
+ raise AssertionError("health check should not run")
+
+ err = handle_rpc_exception_with_health(
+ ConnectException(Code.not_found, "missing"), fail
+ )
+ assert isinstance(err, NotFoundException)
+
+
+async def test_async_health_check_confirms_sandbox_killed():
+ async def check():
+ return False
+
+ err = await ahandle_rpc_exception_with_health(_stream_reset(), check)
+ assert isinstance(err, TimeoutException)
+ assert "sandbox was killed or reached its end of life" in str(err)
+
+
+async def test_async_health_check_failure_returns_raw_error():
+ async def check():
+ raise RuntimeError("health check failed")
+
+ original = _stream_reset()
+ err = await ahandle_rpc_exception_with_health(original, check)
+ assert err is original