Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
chore(js): add max concurrency limits (#1351)
Jakub Novák18a10afa8739
diff --git a/.changeset/cool-camels-remain.md b/.changeset/cool-camels-remain.mdnew file mode 100644index 000000000..e132fc494--- /dev/null+++ b/.changeset/cool-camels-remain.md@@ -0,0 +1,5 @@+---+'e2b': minor+---++Limit max number of inflight connectionsdiff --git a/packages/js-sdk/src/api/http2.ts b/packages/js-sdk/src/api/http2.tsindex 06b321070..bb5e54602 100644--- a/packages/js-sdk/src/api/http2.ts+++ b/packages/js-sdk/src/api/http2.ts@@ -1,4 +1,6 @@import { runtime } from '../utils'+import { parseInflightLimitEnv, parsePositiveIntEnv } from './metadata'+import { limitConcurrency } from './inflight'import {loadUndici,toUndiciRequestInput,@@ -6,7 +8,10 @@ import {type UndiciRequestInit,} from '../undici'-const API_CONNECTION_LIMIT = 100+const DEFAULT_API_CONNECTION_LIMIT = 100+// 1000 = ~10 streams per connection (with the 100-conn default).+// Override via env if your workload needs different.+const DEFAULT_API_INFLIGHT_LIMIT = 1000const API_UNDICI_FALLBACK_WARNING ='Failed to load undici for API HTTP/2 transport; falling back to global fetch.'@@ -27,8 +32,9 @@ export function createApiFetchForRuntime(currentRuntime = runtime,options: {connectionLimit?: number+ inflightLimit?: numberloadUndici?: () => Promise<UndiciModule | undefined>- } = { connectionLimit: API_CONNECTION_LIMIT }+ } = {}): typeof fetch {if (currentRuntime !== 'node') {return fetch@@ -46,27 +52,29 @@ export function createApiFetchForRuntime(async function buildApiFetcher(options: {connectionLimit?: number+ inflightLimit?: numberloadUndici?: () => Promise<UndiciModule | undefined>}): Promise<typeof fetch> {const undici = await (options.loadUndici ?? loadUndici)()+ const inflightLimit = options.inflightLimit ?? getApiInflightLimit()if (!undici) {warnUndiciFallback()- return fetch+ return limitConcurrency(fetch, inflightLimit)}const { Agent, fetch: undiciFetch } = undiciconst dispatcher = new Agent({allowH2: true,- connections: options.connectionLimit,+ connections: options.connectionLimit ?? getApiConnectionLimit(),})const fetchWithDispatcher = undiciFetch as unknown as (input: RequestInfo | URL,init?: UndiciRequestInit) => Promise<Response>- return ((input, init) => {+ const wrapped: typeof fetch = ((input, init) => {const request = toUndiciRequestInput(input, init)return fetchWithDispatcher(request.input, {@@ -74,6 +82,29 @@ async function buildApiFetcher(options: {dispatcher,})}) as typeof fetch++ return limitConcurrency(wrapped, inflightLimit)+}++export function getApiConnectionLimit(): number {+ return parsePositiveIntEnv(+ 'E2B_API_CONNECTIONS',+ DEFAULT_API_CONNECTION_LIMIT+ )+}++/**+ * Returns the configured max number of API requests that can be in flight at+ * once, or `0` to disable the cap.+ *+ * Defaults to {@link DEFAULT_API_INFLIGHT_LIMIT} ({@link 1000}). Override via+ * `E2B_API_INFLIGHT_REQUESTS` env var; set to `0` to disable the cap entirely.+ */+export function getApiInflightLimit(): number {+ return parseInflightLimitEnv(+ 'E2B_API_INFLIGHT_REQUESTS',+ DEFAULT_API_INFLIGHT_LIMIT+ )}function warnUndiciFallback() {diff --git a/packages/js-sdk/src/api/inflight.ts b/packages/js-sdk/src/api/inflight.tsnew file mode 100644index 000000000..3dc41ba72--- /dev/null+++ b/packages/js-sdk/src/api/inflight.ts@@ -0,0 +1,78 @@+/**+ * Simple FIFO semaphore used to cap the number of in-flight requests sent+ * through a fetch dispatcher.+ */+class Semaphore {+ private active = 0+ private readonly queue: Array<() => void> = []++ constructor(private readonly max: number) {}++ async acquire(signal?: AbortSignal): Promise<() => void> {+ if (signal?.aborted) throw abortReason(signal)+ if (this.active < this.max) {+ this.active+++ return () => this.release()+ }++ return new Promise<() => void>((resolve, reject) => {+ const onAcquire = () => {+ signal?.removeEventListener('abort', onAbort)+ this.active+++ resolve(() => this.release())+ }+ const onAbort = () => {+ const i = this.queue.indexOf(onAcquire)+ if (i >= 0) this.queue.splice(i, 1)+ reject(abortReason(signal))+ }+ this.queue.push(onAcquire)+ signal?.addEventListener('abort', onAbort, { once: true })+ })+ }++ private release() {+ this.active--+ const next = this.queue.shift()+ if (next) next()+ }+}++function abortReason(signal: AbortSignal | undefined): unknown {+ return signal?.reason ?? new DOMException('Aborted', 'AbortError')+}++/**+ * Wrap `fetcher` so at most `max` requests are in-flight at any time.+ * Subsequent requests are FIFO-queued inside the SDK process and dispatched+ * as earlier requests settle.+ *+ * NOTE: the slot is released as soon as `fetcher` resolves with the response+ * headers, not when the response body is fully consumed. This means the+ * effective concurrency can be higher than `max` while bodies are+ * still streaming.+ *+ * TODO: release on body end (consume/cancel/error) so the+ * SDK-level cap aligns with the dispatcher's connection accounting+ */+export function limitConcurrency(+ fetcher: typeof fetch,+ max: number+): typeof fetch {+ if (!Number.isFinite(max) || max <= 0) {+ return fetcher+ }++ const sem = new Semaphore(max)++ return (async (input, init) => {+ const signal =+ init?.signal ?? (input instanceof Request ? input.signal : undefined)+ const release = await sem.acquire(signal)+ try {+ return await fetcher(input, init)+ } finally {+ release()+ }+ }) as typeof fetch+}diff --git a/packages/js-sdk/src/api/metadata.ts b/packages/js-sdk/src/api/metadata.tsindex 312863126..b863b1c59 100644--- a/packages/js-sdk/src/api/metadata.ts+++ b/packages/js-sdk/src/api/metadata.ts@@ -27,3 +27,59 @@ export function getEnvVar(name: string) {return process.env[name]}++/**+ * Parse an env var as a base-10 integer, falling back to `defaultValue` when+ * the env var is unset. Throws on non-integer input rather than silently+ * falling back so misconfiguration is surfaced loudly.+ */+export function parseIntEnv(name: string, defaultValue: number): number {+ const raw = getEnvVar(name)+ if (!raw) return defaultValue++ const parsed = Number.parseInt(raw, 10)+ if (!Number.isFinite(parsed)) {+ throw new Error(+ `Invalid ${name}=${JSON.stringify(raw)}: expected an integer.`+ )+ }++ return parsed+}++/**+ * Parse an env var that must be a positive integer (>= 1). Throws on+ * non-positive or non-integer input.+ */+export function parsePositiveIntEnv(+ name: string,+ defaultValue: number+): number {+ const parsed = parseIntEnv(name, defaultValue)+ if (parsed < 1) {+ throw new Error(`Invalid ${name}=${parsed}: expected a positive integer.`)+ }++ return parsed+}++/**+ * Parse an inflight-limit env var. Returns `0` to disable the cap (documented+ * opt-out) or a positive integer to cap concurrency. Throws on non-integer or+ * negative values so misconfiguration is surfaced loudly rather than silently+ * removing the cap. A return value of `0` is recognized by+ * {@link limitConcurrency} as "no cap".+ */+export function parseInflightLimitEnv(+ name: string,+ defaultValue: number+): number {+ const parsed = parseIntEnv(name, defaultValue)+ if (parsed < 0) {+ throw new Error(+ `Invalid ${name}=${parsed}: expected a non-negative integer ` ++ '(use 0 to disable the cap).'+ )+ }+ return parsed+}diff --git a/packages/js-sdk/src/envd/http2.ts b/packages/js-sdk/src/envd/http2.tsindex 29950816d..0b2bda119 100644--- a/packages/js-sdk/src/envd/http2.ts+++ b/packages/js-sdk/src/envd/http2.ts@@ -1,5 +1,6 @@import { runtime } from '../utils'-import { getEnvVar } from '../api/metadata'+import { parseInflightLimitEnv, parsePositiveIntEnv } from '../api/metadata'+import { limitConcurrency } from '../api/inflight'import {loadUndici,toUndiciRequestInput,@@ -9,6 +10,7 @@ import {type EnvdFetchOptions = {connectionLimit?: number+ inflightLimit?: numberloadUndici?: () => Promise<UndiciModule | undefined>}@@ -17,6 +19,8 @@ let envdRpcFetch: typeof fetch | undefinedlet hasWarnedUndiciFallback = falseconst DEFAULT_ENVD_CONNECTION_LIMIT = 10const DEFAULT_ENVD_RPC_CONNECTION_LIMIT = 200+const DEFAULT_ENVD_INFLIGHT_LIMIT = 2000+const DEFAULT_ENVD_RPC_INFLIGHT_LIMIT = 2000export function createEnvdFetchForRuntime(currentRuntime = runtime,@@ -40,11 +44,12 @@ async function buildEnvdFetcher(options: EnvdFetchOptions): Promise<typeof fetch> {const undici = await (options.loadUndici ?? loadUndici)()+ const inflightLimit = options.inflightLimit ?? 0if (!undici) {warnUndiciFallback()- return fetch+ return limitConcurrency(fetch, inflightLimit)}const { Agent, fetch: undiciFetch } = undici@@ -59,7 +64,7 @@ async function buildEnvdFetcher(init?: UndiciRequestInit) => Promise<Response>- return ((input, init) => {+ const wrapped: typeof fetch = ((input, init) => {const request = toUndiciRequestInput(input, init)return fetchWithDispatcher(request.input, {@@ -67,6 +72,8 @@ async function buildEnvdFetcher(dispatcher,})}) as typeof fetch++ return limitConcurrency(wrapped, inflightLimit)}function warnUndiciFallback() {@@ -87,7 +94,9 @@ export function createEnvdFetch(): typeof fetch {// Keep one origin connection for short envd REST calls. If ALPN falls back// to h1, this favors connection pressure over per-sandbox throughput.- envdFetch = createEnvdFetchForRuntime(runtime)+ envdFetch = createEnvdFetchForRuntime(runtime, {+ inflightLimit: getEnvdInflightLimit(),+ })return envdFetch}@@ -99,21 +108,47 @@ export function createEnvdRpcFetch(): typeof fetch {envdRpcFetch = createEnvdFetchForRuntime(runtime, {connectionLimit: getEnvdRpcConnectionLimit(),+ inflightLimit: getEnvdRpcInflightLimit(),})return envdRpcFetch}-export function getEnvdRpcConnectionLimit() {- const raw = getEnvVar('E2B_ENVD_RPC_CONNECTIONS')- if (!raw) {- return DEFAULT_ENVD_RPC_CONNECTION_LIMIT- }+export function getEnvdRpcConnectionLimit(): number {+ return parsePositiveIntEnv(+ 'E2B_ENVD_RPC_CONNECTIONS',+ DEFAULT_ENVD_RPC_CONNECTION_LIMIT+ )+}- const parsed = Number.parseInt(raw, 10)- if (!Number.isFinite(parsed) || parsed < 1) {- return DEFAULT_ENVD_RPC_CONNECTION_LIMIT- }+/**+ * Returns the configured max number of envd REST requests (e.g.+ * `files.read`/`files.write`) that can be in flight at once across all+ * sandboxes in this SDK process, or `0` to disable the cap.+ *+ * Defaults to {@link DEFAULT_ENVD_INFLIGHT_LIMIT} ({@link 2000}). Override+ * via `E2B_ENVD_INFLIGHT_REQUESTS` env var; set to `0` to disable the cap+ * entirely.+ */+export function getEnvdInflightLimit(): number {+ return parseInflightLimitEnv(+ 'E2B_ENVD_INFLIGHT_REQUESTS',+ DEFAULT_ENVD_INFLIGHT_LIMIT+ )+}- return parsed+/**+ * Returns the configured max number of envd RPC requests that+ * can be in flight at once across all sandboxes in this SDK process,+ * or `0` to disable the cap.+ *+ * Defaults to {@link DEFAULT_ENVD_RPC_INFLIGHT_LIMIT} ({@link 2000}). Override+ * via `E2B_ENVD_RPC_INFLIGHT_REQUESTS` env var; set to `0` to disable the cap+ * entirely.+ */+export function getEnvdRpcInflightLimit(): number {+ return parseInflightLimitEnv(+ 'E2B_ENVD_RPC_INFLIGHT_REQUESTS',+ DEFAULT_ENVD_RPC_INFLIGHT_LIMIT+ )}diff --git a/packages/js-sdk/tests/api/http2.test.ts b/packages/js-sdk/tests/api/http2.test.tsindex d19498ed4..6fb203c00 100644--- a/packages/js-sdk/tests/api/http2.test.ts+++ b/packages/js-sdk/tests/api/http2.test.ts@@ -5,6 +5,8 @@ afterEach(() => {vi.resetModules()vi.doUnmock('undici')vi.doUnmock('../../src/utils')+ delete process.env.E2B_API_CONNECTIONS+ delete process.env.E2B_API_INFLIGHT_REQUESTS})test('uses undici with a bounded HTTP/2 dispatcher for API requests', async () => {@@ -35,3 +37,35 @@ test('uses undici with a bounded HTTP/2 dispatcher for API requests', async () =expect(agents).toEqual([{ allowH2: true, connections: 100 }])expect(requests[0].init?.dispatcher).toBeInstanceOf(Agent)})++test('getApiConnectionLimit throws on a malformed env value', async () => {+ process.env.E2B_API_CONNECTIONS = 'not-a-number'++ const { getApiConnectionLimit } = await import('../../src/api/http2')++ expect(() => getApiConnectionLimit()).toThrow(/E2B_API_CONNECTIONS/)+})++test('getApiInflightLimit throws on a malformed env value', async () => {+ process.env.E2B_API_INFLIGHT_REQUESTS = 'not-a-number'++ const { getApiInflightLimit } = await import('../../src/api/http2')++ expect(() => getApiInflightLimit()).toThrow(/E2B_API_INFLIGHT_REQUESTS/)+})++test('getApiInflightLimit returns 0 when explicitly disabled', async () => {+ process.env.E2B_API_INFLIGHT_REQUESTS = '0'++ const { getApiInflightLimit } = await import('../../src/api/http2')++ expect(getApiInflightLimit()).toBe(0)+})++test('getApiInflightLimit throws on negative env value', async () => {+ process.env.E2B_API_INFLIGHT_REQUESTS = '-5'++ const { getApiInflightLimit } = await import('../../src/api/http2')++ expect(() => getApiInflightLimit()).toThrow(/E2B_API_INFLIGHT_REQUESTS=-5/)+})diff --git a/packages/js-sdk/tests/api/inflight.test.ts b/packages/js-sdk/tests/api/inflight.test.tsnew file mode 100644index 000000000..f9ea00bdd--- /dev/null+++ b/packages/js-sdk/tests/api/inflight.test.ts@@ -0,0 +1,75 @@+import { expect, test, vi } from 'vitest'++import { limitConcurrency } from '../../src/api/inflight'++function deferred<T>() {+ let resolve!: (value: T) => void+ let reject!: (reason?: unknown) => void+ const promise = new Promise<T>((res, rej) => {+ resolve = res+ reject = rej+ })+ return { promise, resolve, reject }+}++test('limitConcurrency queues requests over the cap and releases on response', async () => {+ const gate = deferred<Response>()+ let secondStarted = false+ const inner = vi.fn(async (input: RequestInfo | URL) => {+ if (String(input).endsWith('/first')) return gate.promise+ secondStarted = true+ return new Response('second')+ }) as unknown as typeof fetch++ const limited = limitConcurrency(inner, 1)+ const first = limited('https://example.com/first')+ const second = limited('https://example.com/second')++ await Promise.resolve()+ await Promise.resolve()+ expect(secondStarted).toBe(false)++ gate.resolve(new Response('first'))+ expect(await (await first).text()).toBe('first')+ expect(await (await second).text()).toBe('second')+ expect(secondStarted).toBe(true)+})++test('limitConcurrency releases when the underlying fetch rejects', async () => {+ let calls = 0+ const inner = vi.fn(async () => {+ calls+++ if (calls === 1) throw new Error('boom')+ return new Response('ok')+ }) as unknown as typeof fetch++ const limited = limitConcurrency(inner, 1)+ await expect(limited('https://example.com/a')).rejects.toThrow('boom')++ // Slot should be free for the next request.+ const res = await limited('https://example.com/b')+ expect(await res.text()).toBe('ok')+})++test('limitConcurrency aborts queued requests when their signal fires', async () => {+ const gate = deferred<Response>()+ const inner = vi.fn(async () => gate.promise) as unknown as typeof fetch+ const limited = limitConcurrency(inner, 1)++ // Occupy the only slot.+ const first = limited('https://example.com/first')++ const controller = new AbortController()+ const queued = limited('https://example.com/queued', {+ signal: controller.signal,+ })++ // Abort the queued request before the slot frees.+ controller.abort()+ await expect(queued).rejects.toMatchObject({ name: 'AbortError' })++ // Release the first request to make sure cleanup did not break the slot.+ gate.resolve(new Response('done'))+ const resp = await first+ expect(await resp.text()).toBe('done')+})diff --git a/packages/js-sdk/tests/envd/http2.test.ts b/packages/js-sdk/tests/envd/http2.test.tsindex c73092e96..8e4e7eef5 100644--- a/packages/js-sdk/tests/envd/http2.test.ts+++ b/packages/js-sdk/tests/envd/http2.test.ts@@ -6,6 +6,8 @@ afterEach(() => {vi.doUnmock('undici')vi.doUnmock('../../src/utils')delete process.env.E2B_ENVD_RPC_CONNECTIONS+ delete process.env.E2B_ENVD_INFLIGHT_REQUESTS+ delete process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS})test('uses undici with HTTP/2 enabled in Node', async () => {@@ -97,6 +99,62 @@ test('reads RPC stream dispatcher connection limit from env', async () => {expect(getEnvdRpcConnectionLimit()).toBe(200)})+test('getEnvdRpcConnectionLimit throws on malformed env value', async () => {+ process.env.E2B_ENVD_RPC_CONNECTIONS = 'bogus'++ const { getEnvdRpcConnectionLimit } = await import('../../src/envd/http2')++ expect(() => getEnvdRpcConnectionLimit()).toThrow(/E2B_ENVD_RPC_CONNECTIONS/)+})++test('getEnvdInflightLimit throws on malformed env value', async () => {+ process.env.E2B_ENVD_INFLIGHT_REQUESTS = 'bogus'++ const { getEnvdInflightLimit } = await import('../../src/envd/http2')++ expect(() => getEnvdInflightLimit()).toThrow(/E2B_ENVD_INFLIGHT_REQUESTS/)+})++test('getEnvdRpcInflightLimit throws on malformed env value', async () => {+ process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS = 'bogus'++ const { getEnvdRpcInflightLimit } = await import('../../src/envd/http2')++ expect(() => getEnvdRpcInflightLimit()).toThrow(+ /E2B_ENVD_RPC_INFLIGHT_REQUESTS/+ )+})++test('inflight limit env vars return 0 when explicitly disabled', async () => {+ process.env.E2B_ENVD_INFLIGHT_REQUESTS = '0'+ process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS = '0'++ const { getEnvdInflightLimit, getEnvdRpcInflightLimit } = await import(+ '../../src/envd/http2'+ )++ expect(getEnvdInflightLimit()).toBe(0)+ expect(getEnvdRpcInflightLimit()).toBe(0)+})++test('getEnvdInflightLimit throws on negative env value', async () => {+ process.env.E2B_ENVD_INFLIGHT_REQUESTS = '-1'++ const { getEnvdInflightLimit } = await import('../../src/envd/http2')++ expect(() => getEnvdInflightLimit()).toThrow(/E2B_ENVD_INFLIGHT_REQUESTS=-1/)+})++test('getEnvdRpcInflightLimit throws on negative env value', async () => {+ process.env.E2B_ENVD_RPC_INFLIGHT_REQUESTS = '-5'++ const { getEnvdRpcInflightLimit } = await import('../../src/envd/http2')++ expect(() => getEnvdRpcInflightLimit()).toThrow(+ /E2B_ENVD_RPC_INFLIGHT_REQUESTS=-5/+ )+})+test('defers loading undici until the first Node request', async () => {const Agent = vi.fn()const undiciFetch = vi.fn(() => Promise.resolve(new Response('ok')))