Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

chore(js): add max concurrency limits (#1351)

Jakub Novák18a10afa8739
diff --git a/.changeset/cool-camels-remain.md b/.changeset/cool-camels-remain.md
new file mode 100644
index 000000000..e132fc494
--- /dev/null
+++ b/.changeset/cool-camels-remain.md
@@ -0,0 +1,5 @@
+---
+'e2b': minor
+---
+
+Limit max number of inflight connections
diff --git a/packages/js-sdk/src/api/http2.ts b/packages/js-sdk/src/api/http2.ts
index 06b321070..bb5e54602 100644
--- a/packages/js-sdk/src/api/http2.ts
+++ b/packages/js-sdk/src/api/http2.ts
@@ -1,4 +1,6 @@
import { runtime } from '../utils'
+import { parseInflightLimitEnv, parsePositiveIntEnv } from './metadata'
+import { limitConcurrency } from './inflight'
import {
loadUndici,
toUndiciRequestInput,
@@ -6,7 +8,10 @@ import {
type UndiciRequestInit,
} from '../undici'
-const API_CONNECTION_LIMIT = 100
+const DEFAULT_API_CONNECTION_LIMIT = 100
+// 1000 = ~10 streams per connection (with the 100-conn default).
+// Override via env if your workload needs different.
+const DEFAULT_API_INFLIGHT_LIMIT = 1000
const API_UNDICI_FALLBACK_WARNING =
'Failed to load undici for API HTTP/2 transport; falling back to global fetch.'
@@ -27,8 +32,9 @@ export function createApiFetchForRuntime(
currentRuntime = runtime,
options: {
connectionLimit?: number
+ inflightLimit?: number
loadUndici?: () => Promise<UndiciModule | undefined>
- } = { connectionLimit: API_CONNECTION_LIMIT }
+ } = {}
): typeof fetch {
if (currentRuntime !== 'node') {
return fetch
@@ -46,27 +52,29 @@ export function createApiFetchForRuntime(
async function buildApiFetcher(options: {
connectionLimit?: number
+ inflightLimit?: number
loadUndici?: () => Promise<UndiciModule | undefined>
}): Promise<typeof fetch> {
const undici = await (options.loadUndici ?? loadUndici)()
+ const inflightLimit = options.inflightLimit ?? getApiInflightLimit()
if (!undici) {
warnUndiciFallback()
- return fetch
+ return limitConcurrency(fetch, inflightLimit)
}
const { Agent, fetch: undiciFetch } = undici
const dispatcher = new Agent({
allowH2: true,
- connections: options.connectionLimit,
+ connections: options.connectionLimit ?? getApiConnectionLimit(),
})
const fetchWithDispatcher = undiciFetch as unknown as (
input: RequestInfo | URL,
init?: UndiciRequestInit
) => Promise<Response>
- return ((input, init) => {
+ const wrapped: typeof fetch = ((input, init) => {
const request = toUndiciRequestInput(input, init)
return fetchWithDispatcher(request.input, {
@@ -74,6 +82,29 @@ async function buildApiFetcher(options: {
dispatcher,
})
}) as typeof fetch
+
+ return limitConcurrency(wrapped, inflightLimit)
+}
+
+export function getApiConnectionLimit(): number {
+ return parsePositiveIntEnv(
+ 'E2B_API_CONNECTIONS',
+ DEFAULT_API_CONNECTION_LIMIT
+ )
+}
+
+/**
+ * Returns the configured max number of API requests that can be in flight at
+ * once, or `0` to disable the cap.
+ *
+ * Defaults to {@link DEFAULT_API_INFLIGHT_LIMIT} ({@link 1000}). Override via
+ * `E2B_API_INFLIGHT_REQUESTS` env var; set to `0` to disable the cap entirely.
+ */
+export function getApiInflightLimit(): number {
+ return parseInflightLimitEnv(
+ 'E2B_API_INFLIGHT_REQUESTS',
+ DEFAULT_API_INFLIGHT_LIMIT
+ )
}
function warnUndiciFallback() {
diff --git a/packages/js-sdk/src/api/inflight.ts b/packages/js-sdk/src/api/inflight.ts
new file mode 100644
index 000000000..3dc41ba72
--- /dev/null
+++ b/packages/js-sdk/src/api/inflight.ts
@@ -0,0 +1,78 @@
+/**
+ * Simple FIFO semaphore used to cap the number of in-flight requests sent
+ * through a fetch dispatcher.
+ */
+class Semaphore {
+ private active = 0
+ private readonly queue: Array<() => void> = []
+
+ constructor(private readonly max: number) {}
+
+ async acquire(signal?: AbortSignal): Promise<() => void> {
+ if (signal?.aborted) throw abortReason(signal)
+ if (this.active < this.max) {
+ this.active++
+ return () => this.release()
+ }
+
+ return new Promise<() => void>((resolve, reject) => {
+ const onAcquire = () => {
+ signal?.removeEventListener('abort', onAbort)
+ this.active++
+ resolve(() => this.release())
+ }
+ const onAbort = () => {
+ const i = this.queue.indexOf(onAcquire)
+ if (i >= 0) this.queue.splice(i, 1)
+ reject(abortReason(signal))
+ }
+ this.queue.push(onAcquire)
+ signal?.addEventListener('abort', onAbort, { once: true })
+ })
+ }
+
+ private release() {
+ this.active--
+ const next = this.queue.shift()
+ if (next) next()
+ }
+}
+
+function abortReason(signal: AbortSignal | undefined): unknown {
+ return signal?.reason ?? new DOMException('Aborted', 'AbortError')
+}
+
+/**
+ * Wrap `fetcher` so at most `max` requests are in-flight at any time.
+ * Subsequent requests are FIFO-queued inside the SDK process and dispatched
+ * as earlier requests settle.
+ *
+ * NOTE: the slot is released as soon as `fetcher` resolves with the response
+ * headers, not when the response body is fully consumed. This means the
+ * effective concurrency can be higher than `max` while bodies are
+ * still streaming.
+ *
+ * TODO: release on body end (consume/cancel/error) so the
+ * SDK-level cap aligns with the dispatcher's connection accounting
+ */
+export function limitConcurrency(
+ fetcher: typeof fetch,
+ max: number
+): typeof fetch {
+ if (!Number.isFinite(max) || max <= 0) {
+ return fetcher
+ }
+
+ const sem = new Semaphore(max)
+
+ return (async (input, init) => {
+ const signal =
+ init?.signal ?? (input instanceof Request ? input.signal : undefined)
+ const release = await sem.acquire(signal)
+ try {
+ return await fetcher(input, init)
+ } finally {
+ release()
+ }
+ }) as typeof fetch
+}
diff --git a/packages/js-sdk/src/api/metadata.ts b/packages/js-sdk/src/api/metadata.ts
index 312863126..b863b1c59 100644
--- a/packages/js-sdk/src/api/metadata.ts
+++ b/packages/js-sdk/src/api/metadata.ts
@@ -27,3 +27,59 @@ export function getEnvVar(name: string) {
return process.env[name]
}
+
+/**
+ * Parse an env var as a base-10 integer, falling back to `defaultValue` when
+ * the env var is unset. Throws on non-integer input rather than silently
+ * falling back so misconfiguration is surfaced loudly.
+ */
+export function parseIntEnv(name: string, defaultValue: number): number {
+ const raw = getEnvVar(name)
+ if (!raw) return defaultValue
+
+ const parsed = Number.parseInt(raw, 10)
+ if (!Number.isFinite(parsed)) {
+ throw new Error(
+ `Invalid ${name}=${JSON.stringify(raw)}: expected an integer.`
+ )
+ }
+
+ return parsed
+}
+
+/**
+ * Parse an env var that must be a positive integer (>= 1). Throws on
+ * non-positive or non-integer input.
+ */
+export function parsePositiveIntEnv(
+ name: string,
+ defaultValue: number
+): number {
+ const parsed = parseIntEnv(name, defaultValue)
+ if (parsed < 1) {
+ throw new Error(`Invalid ${name}=${parsed}: expected a positive integer.`)
+ }
+
+ return parsed
+}
+
+/**
+ * Parse an inflight-limit env var. Returns `0` to disable the cap (documented
+ * opt-out) or a positive integer to cap concurrency. Throws on non-integer or
+ * negative values so misconfiguration is surfaced loudly rather than silently
+ * removing the cap. A return value of `0` is recognized by
+ * {@link limitConcurrency} as "no cap".
+ */
+export function parseInflightLimitEnv(
+ name: string,
+ defaultValue: number
+): number {
+ const parsed = parseIntEnv(name, defaultValue)
+ if (parsed < 0) {
+ throw new Error(
+ `Invalid ${name}=${parsed}: expected a non-negative integer ` +
+ '(use 0 to disable the cap).'
+ )
+ }
+ return parsed
+}
diff --git a/packages/js-sdk/src/envd/http2.ts b/packages/js-sdk/src/envd/http2.ts
index 29950816d..0b2bda119 100644
--- a/packages/js-sdk/src/envd/http2.ts
+++ b/packages/js-sdk/src/envd/http2.ts
@@ -1,5 +1,6 @@
import { runtime } from '../utils'
-import { getEnvVar } from '../api/metadata'
+import { parseInflightLimitEnv, parsePositiveIntEnv } from '../api/metadata'
+import { limitConcurrency } from '../api/inflight'
import {
loadUndici,
toUndiciRequestInput,
@@ -9,6 +10,7 @@ import {
type EnvdFetchOptions = {
connectionLimit?: number
+ inflightLimit?: number
loadUndici?: () => Promise<UndiciModule | undefined>
}
@@ -17,6 +19,8 @@ let envdRpcFetch: typeof fetch | undefined
let hasWarnedUndiciFallback = false
const DEFAULT_ENVD_CONNECTION_LIMIT = 10
const DEFAULT_ENVD_RPC_CONNECTION_LIMIT = 200
+const DEFAULT_ENVD_INFLIGHT_LIMIT = 2000
+const DEFAULT_ENVD_RPC_INFLIGHT_LIMIT = 2000
export function createEnvdFetchForRuntime(
currentRuntime = runtime,
@@ -40,11 +44,12 @@ async function buildEnvdFetcher(
options: EnvdFetchOptions
): Promise<typeof fetch> {
const undici = await (options.loadUndici ?? loadUndici)()
+ const inflightLimit = options.inflightLimit ?? 0
if (!undici) {
warnUndiciFallback()
- return fetch
+ return limitConcurrency(fetch, inflightLimit)
}
const { Agent, fetch: undiciFetch } = undici
@@ -59,7 +64,7 @@ async function buildEnvdFetcher(
init?: UndiciRequestInit
) => Promise<Response>
- return ((input, init) => {
+ const wrapped: typeof fetch = ((input, init) => {
const request = toUndiciRequestInput(input, init)
return fetchWithDispatcher(request.input, {
@@ -67,6 +72,8 @@ async function buildEnvdFetcher(
dispatcher,
})
}) as typeof fetch
+
+ return limitConcurrency(wrapped, inflightLimit)
}
function warnUndiciFallback() {
@@ -87,7 +94,9 @@ export function createEnvdFetch(): typeof fetch {
// Keep one origin connection for short envd REST calls. If ALPN falls back
// to h1, this favors connection pressure over per-sandbox throughput.
- envdFetch = createEnvdFetchForRuntime(runtime)
+ envdFetch = createEnvdFetchForRuntime(runtime, {
+ inflightLimit: getEnvdInflightLimit(),
+ })
return envdFetch
}
@@ -99,21 +108,47 @@ export function createEnvdRpcFetch(): typeof fetch {
envdRpcFetch = createEnvdFetchForRuntime(runtime, {
connectionLimit: getEnvdRpcConnectionLimit(),
+ inflightLimit: getEnvdRpcInflightLimit(),
})
return envdRpcFetch
}
-export function getEnvdRpcConnectionLimit() {
- const raw = getEnvVar('E2B_ENVD_RPC_CONNECTIONS')
- if (!raw) {
- return DEFAULT_ENVD_RPC_CONNECTION_LIMIT
- }
+export function getEnvdRpcConnectionLimit(): number {
+ return parsePositiveIntEnv(
+ 'E2B_ENVD_RPC_CONNECTIONS',
+ DEFAULT_ENVD_RPC_CONNECTION_LIMIT
+ )
+}
- const parsed = Number.parseInt(raw, 10)
- if (!Number.isFinite(parsed) || parsed < 1) {
- return DEFAULT_ENVD_RPC_CONNECTION_LIMIT
- }
+/**
+ * Returns the configured max number of envd REST requests (e.g.
+ * `files.read`/`files.write`) that can be in flight at once across all
+ * sandboxes in this SDK process, or `0` to disable the cap.
+ *
+ * Defaults to {@link DEFAULT_ENVD_INFLIGHT_LIMIT} ({@link 2000}). Override
+ * via `E2B_ENVD_INFLIGHT_REQUESTS` env var; set to `0` to disable the cap
+ * entirely.
+ */
+export function getEnvdInflightLimit(): number {
+ return parseInflightLimitEnv(
+ 'E2B_ENVD_INFLIGHT_REQUESTS',
+ DEFAULT_ENVD_INFLIGHT_LIMIT
+ )
+}
- return parsed
+/**
+ * Returns the configured max number of envd RPC requests that
+ * can be in flight at once across all sandboxes in this SDK process,
+ * or `0` to disable the cap.
+ *
+ * Defaults to {@link DEFAULT_ENVD_RPC_INFLIGHT_LIMIT} ({@link 2000}). Override
+ * via `E2B_ENVD_RPC_INFLIGHT_REQUESTS` env var; set to `0` to disable the cap
+ * entirely.
+ */
+export function getEnvdRpcInflightLimit(): number {
+ return parseInflightLimitEnv(
+ 'E2B_ENVD_RPC_INFLIGHT_REQUESTS',
+ DEFAULT_ENVD_RPC_INFLIGHT_LIMIT
+ )
}
diff --git a/packages/js-sdk/tests/api/http2.test.ts b/packages/js-sdk/tests/api/http2.test.ts
index d19498ed4..6fb203c00 100644
--- a/packages/js-sdk/tests/api/http2.test.ts
+++ b/packages/js-sdk/tests/api/http2.test.ts
@@ -5,6 +5,8 @@ afterEach(() => {
vi.resetModules()
vi.doUnmock('undici')
vi.doUnmock('../../src/utils')
+ delete process.env.E2B_API_CONNECTIONS
+ delete process.env.E2B_API_INFLIGHT_REQUESTS
})
test('uses undici with a bounded HTTP/2 dispatcher for API requests', async () => {
@@ -35,3 +37,35 @@ test('uses undici with a bounded HTTP/2 dispatcher for API requests', async () =
expect(agents).toEqual([{ allowH2: true, connections: 100 }])
expect(requests[0].init?.dispatcher).toBeInstanceOf(Agent)
})
+
+test('getApiConnectionLimit throws on a malformed env value', async () => {
+ process.env.E2B_API_CONNECTIONS = 'not-a-number'
+
+ const { getApiConnectionLimit } = await import('../../src/api/http2')
+
+ expect(() => getApiConnectionLimit()).toThrow(/E2B_API_CONNECTIONS/)
+})
+
+test('getApiInflightLimit throws on a malformed env value', async () => {
+ process.env.E2B_API_INFLIGHT_REQUESTS = 'not-a-number'
+
+ const { getApiInflightLimit } = await import('../../src/api/http2')
+
+ expect(() => getApiInflightLimit()).toThrow(/E2B_API_INFLIGHT_REQUESTS/)
+})
+
+test('getApiInflightLimit returns 0 when explicitly disabled', async () => {
+ process.env.E2B_API_INFLIGHT_REQUESTS = '0'
+
+ const { getApiInflightLimit } = await import('../../src/api/http2')
+
+ expect(getApiInflightLimit()).toBe(0)
+})
+
+test('getApiInflightLimit throws on negative env value', async () => {
+ process.env.E2B_API_INFLIGHT_REQUESTS = '-5'
+
+ const { getApiInflightLimit } = await import('../../src/api/http2')
+
+ expect(() => getApiInflightLimit()).toThrow(/E2B_API_INFLIGHT_REQUESTS=-5/)
+})
diff --git a/packages/js-sdk/tests/api/inflight.test.ts b/packages/js-sdk/tests/api/inflight.test.ts
new file mode 100644
index 000000000..f9ea00bdd
--- /dev/null
+++ b/packages/js-sdk/tests/api/inflight.test.ts
@@ -0,0 +1,75 @@
+import { expect, test, vi } from 'vitest'
+
+import { limitConcurrency } from '../../src/api/inflight'
+
+function deferred<T>() {
+ let resolve!: (value: T) => void
+ let reject!: (reason?: unknown) => void
+ const promise = new Promise<T>((res, rej) => {
+ resolve = res
+ reject = rej
+ })
+ return { promise, resolve, reject }
+}
+
+test('limitConcurrency queues requests over the cap and releases on response', async () => {
+ const gate = deferred<Response>()
+ let secondStarted = false
+ const inner = vi.fn(async (input: RequestInfo | URL) => {
+ if (String(input).endsWith('/first')) return gate.promise
+ secondStarted = true
+ return new Response('second')
+ }) as unknown as typeof fetch
+
+ const limited = limitConcurrency(inner, 1)
+ const first = limited('https://example.com/first')
+ const second = limited('https://example.com/second')
+
+ await Promise.resolve()
+ await Promise.resolve()
+ expect(secondStarted).toBe(false)
+
+ gate.resolve(new Response('first'))
+ expect(await (await first).text()).toBe('first')
+ expect(await (await second).text()).toBe('second')
+ expect(secondStarted).toBe(true)
+})
+
+test('limitConcurrency releases when the underlying fetch rejects', async () => {
+ let calls = 0
+ const inner = vi.fn(async () => {
+ calls++
+ if (calls === 1) throw new Error('boom')
+ return new Response('ok')
+ }) as unknown as typeof fetch
+
+ const limited = limitConcurrency(inner, 1)
+ await expect(limited('https://example.com/a')).rejects.toThrow('boom')
+
+ // Slot should be free for the next request.
+ const res = await limited('https://example.com/b')
+ expect(await res.text()).toBe('ok')
+})
+
+test('limitConcurrency aborts queued requests when their signal fires', async () => {
+ const gate = deferred<Response>()
+ const inner = vi.fn(async () => gate.promise) as unknown as typeof fetch
+ const limited = limitConcurrency(inner, 1)
+
+ // Occupy the only slot.
+ const first = limited('https://example.com/first')
+
+ const controller = new AbortController()
+ const queued = limited('https://example.com/queued', {
+ signal: controller.signal,
+ })
+
+ // Abort the queued request before the slot frees.
+ controller.abort()
+ await expect(queued).rejects.toMatchObject({ name: 'AbortError' })
+
+ // Release the first request to make sure cleanup did not break the slot.
+ gate.resolve(new Response('done'))
+ const resp = await first
+ expect(await resp.text()).toBe('done')
+})
diff --git a/packages/js-sdk/tests/envd/http2.test.ts b/packages/js-sdk/tests/envd/http2.test.ts
index c73092e96..8e4e7eef5 100644
--- a/packages/js-sdk/tests/envd/http2.test.ts
+++ b/packages/js-sdk/tests/envd/http2.test.ts
@@ -6,6 +6,8 @@ afterEach(() => {
vi.doUnmock('undici')
vi.doUnmock('../../src/utils')
delete process.env.E2B_ENVD_RPC_CONNECTIONS
+ delete process.env.E2B_ENVD_INFLIGHT_REQUESTS
+ delete process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS
})
test('uses undici with HTTP/2 enabled in Node', async () => {
@@ -97,6 +99,62 @@ test('reads RPC stream dispatcher connection limit from env', async () => {
expect(getEnvdRpcConnectionLimit()).toBe(200)
})
+test('getEnvdRpcConnectionLimit throws on malformed env value', async () => {
+ process.env.E2B_ENVD_RPC_CONNECTIONS = 'bogus'
+
+ const { getEnvdRpcConnectionLimit } = await import('../../src/envd/http2')
+
+ expect(() => getEnvdRpcConnectionLimit()).toThrow(/E2B_ENVD_RPC_CONNECTIONS/)
+})
+
+test('getEnvdInflightLimit throws on malformed env value', async () => {
+ process.env.E2B_ENVD_INFLIGHT_REQUESTS = 'bogus'
+
+ const { getEnvdInflightLimit } = await import('../../src/envd/http2')
+
+ expect(() => getEnvdInflightLimit()).toThrow(/E2B_ENVD_INFLIGHT_REQUESTS/)
+})
+
+test('getEnvdRpcInflightLimit throws on malformed env value', async () => {
+ process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS = 'bogus'
+
+ const { getEnvdRpcInflightLimit } = await import('../../src/envd/http2')
+
+ expect(() => getEnvdRpcInflightLimit()).toThrow(
+ /E2B_ENVD_RPC_INFLIGHT_REQUESTS/
+ )
+})
+
+test('inflight limit env vars return 0 when explicitly disabled', async () => {
+ process.env.E2B_ENVD_INFLIGHT_REQUESTS = '0'
+ process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS = '0'
+
+ const { getEnvdInflightLimit, getEnvdRpcInflightLimit } = await import(
+ '../../src/envd/http2'
+ )
+
+ expect(getEnvdInflightLimit()).toBe(0)
+ expect(getEnvdRpcInflightLimit()).toBe(0)
+})
+
+test('getEnvdInflightLimit throws on negative env value', async () => {
+ process.env.E2B_ENVD_INFLIGHT_REQUESTS = '-1'
+
+ const { getEnvdInflightLimit } = await import('../../src/envd/http2')
+
+ expect(() => getEnvdInflightLimit()).toThrow(/E2B_ENVD_INFLIGHT_REQUESTS=-1/)
+})
+
+test('getEnvdRpcInflightLimit throws on negative env value', async () => {
+ process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS = '-5'
+
+ const { getEnvdRpcInflightLimit } = await import('../../src/envd/http2')
+
+ expect(() => getEnvdRpcInflightLimit()).toThrow(
+ /E2B_ENVD_RPC_INFLIGHT_REQUESTS=-5/
+ )
+})
+
test('defers loading undici until the first Node request', async () => {
const Agent = vi.fn()
const undiciFetch = vi.fn(() => Promise.resolve(new Response('ok')))