Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
chore(cli): tighten up sandbox exec piping tests (#1129)
Matt Brockmanbb6676f36cca
diff --git a/packages/cli/tests/commands/sandbox/backend_integration.test.ts b/packages/cli/tests/commands/sandbox/backend_integration.test.tsindex 0e46225b2..13ab3084d 100644--- a/packages/cli/tests/commands/sandbox/backend_integration.test.ts+++ b/packages/cli/tests/commands/sandbox/backend_integration.test.ts@@ -1,8 +1,13 @@-import { spawn, spawnSync } from 'node:child_process'-import path from 'node:path'import { afterAll, beforeAll, describe, expect, test } from 'vitest'import { Sandbox } from 'e2b'import { getUserConfig } from 'src/user'+import {+ bufferToText,+ isDebug,+ parseEnvInt,+ runCli,+ runCliWithPipedStdin,+} from '../../setup'type UserConfigWithDomain = NonNullable<ReturnType<typeof getUserConfig>> & {domain?: string@@ -16,21 +21,30 @@ const domain =userConfig?.domain ||'e2b.app'const apiKey = process.env.E2B_API_KEY || userConfig?.teamApiKey+const shouldSkip = !apiKey || isDebug+const integrationTest = test.skipIf(shouldSkip)const templateId =process.env.E2B_CLI_BACKEND_TEMPLATE_ID ||process.env.E2B_TEMPLATE_ID ||'base'-const isDebug = process.env.E2B_DEBUG !== undefined-const hasCreds = Boolean(apiKey)-const shouldSkip = !hasCreds || isDebug-const testIf = test.skipIf(shouldSkip)-const cliPath = path.join(process.cwd(), 'dist', 'index.js')const sandboxTimeoutMs = parseEnvInt('E2B_CLI_BACKEND_SANDBOX_TIMEOUT_MS',20_000)const perTestTimeoutMs = parseEnvInt('E2B_CLI_BACKEND_TEST_TIMEOUT_MS', 30_000)const spawnTimeoutMs = perTestTimeoutMs+const cliEnv: NodeJS.ProcessEnv = {+ ...process.env,+ E2B_DOMAIN: domain,+ E2B_API_KEY: apiKey,+}++delete cliEnv.E2B_DEBUG++const runCliInSandbox = (args: string[]) =>+ runCli(args, { timeoutMs: spawnTimeoutMs, env: cliEnv })+const runCliWithPipeInSandbox = (args: string[], input: Buffer) =>+ runCliWithPipedStdin(args, input, { timeoutMs: spawnTimeoutMs, env: cliEnv })describe('sandbox cli backend integration', () => {let sandbox: Sandbox@@ -57,17 +71,21 @@ describe('sandbox cli backend integration', () => {}}, 15_000)- testIf('list shows the sandbox', { timeout: perTestTimeoutMs }, async () => {- const listResult = runCli(['sandbox', 'list', '--format', 'json'])- expect(listResult.status).toBe(0)- expect(sandboxExistsInList(listResult.stdout, sandbox.sandboxId)).toBe(true)- })+ integrationTest(+ 'list shows the sandbox',+ { timeout: perTestTimeoutMs },+ async () => {+ const listResult = runCliInSandbox(['sandbox', 'list', '--format', 'json'])+ expect(listResult.status).toBe(0)+ expect(sandboxExistsInList(listResult.stdout, sandbox.sandboxId)).toBe(true)+ }+ )- testIf(+ integrationTest('info shows the sandbox details',{ timeout: perTestTimeoutMs },async () => {- const infoResult = runCli([+ const infoResult = runCliInSandbox(['sandbox','info',sandbox.sandboxId,@@ -86,11 +104,11 @@ describe('sandbox cli backend integration', () => {})- testIf(+ integrationTest('exec runs a command without piped stdin',{ timeout: perTestTimeoutMs },async () => {- const execResult = runCli([+ const execResult = runCliInSandbox(['sandbox','exec',sandbox.sandboxId,@@ -104,11 +122,11 @@ describe('sandbox cli backend integration', () => {})- testIf(+ integrationTest('exec runs a command with piped stdin',{ timeout: perTestTimeoutMs },async () => {- const pipedExecResult = await runCliWithPipedStdin(+ const pipedExecResult = await runCliWithPipeInSandbox(['sandbox', 'exec', sandbox.sandboxId, '--', 'sh', '-lc', 'wc -c'],Buffer.from('hello\n', 'utf8'))@@ -120,27 +138,11 @@ describe('sandbox cli backend integration', () => {})- /** Note: removing this test for now because it can be slow to get the logs causing tests to time out */- // testIf(- // 'logs returns successfully',- // { timeout: perTestTimeoutMs },- // async () => {- // const logsResult = runCli([- // 'sandbox',- // 'logs',- // sandbox.sandboxId,- // '--format',- // 'json',- // ])- // expect(logsResult.status).toBe(0)- // }- // )-- testIf(+ integrationTest('metrics returns successfully',{ timeout: perTestTimeoutMs },async () => {- const metricsResult = runCli([+ const metricsResult = runCliInSandbox(['sandbox','metrics',sandbox.sandboxId,@@ -151,11 +153,11 @@ describe('sandbox cli backend integration', () => {})- testIf(+ integrationTest('kill removes the sandbox',{ timeout: perTestTimeoutMs },async () => {- const killResult = runCli(['sandbox', 'kill', sandbox.sandboxId])+ const killResult = runCliInSandbox(['sandbox', 'kill', sandbox.sandboxId])expect(killResult.status).toBe(0)await assertSandboxNotListed(sandbox.sandboxId)@@ -163,90 +165,12 @@ describe('sandbox cli backend integration', () => {)})-function runCli(- args: string[],- opts?: { input?: string | Buffer }-): ReturnType<typeof spawnSync> {- const env: NodeJS.ProcessEnv = {- ...process.env,- E2B_DOMAIN: domain,- E2B_API_KEY: apiKey,- }- delete env.E2B_DEBUG-- return spawnSync('node', [cliPath, ...args], {- env,- input: opts?.input,- encoding: 'utf8',- timeout: spawnTimeoutMs,- })-}--type PipeRunResult = {- status: number | null- stdout: Buffer- stderr: Buffer- error?: Error-}--function runCliWithPipedStdin(- args: string[],- input: Buffer-): Promise<PipeRunResult> {- const env: NodeJS.ProcessEnv = {- ...process.env,- E2B_DOMAIN: domain,- E2B_API_KEY: apiKey,- }- delete env.E2B_DEBUG-- return new Promise((resolve) => {- const child = spawn('node', [cliPath, ...args], {- env,- stdio: ['pipe', 'pipe', 'pipe'],- })-- const stdoutChunks: Buffer[] = []- const stderrChunks: Buffer[] = []- let childError: Error | undefined- let timedOut = false-- const timer = setTimeout(() => {- timedOut = true- child.kill()- }, spawnTimeoutMs)-- child.stdout.on('data', (chunk) => stdoutChunks.push(Buffer.from(chunk)))- child.stderr.on('data', (chunk) => stderrChunks.push(Buffer.from(chunk)))- child.on('error', (err) => {- childError = err- })- child.on('close', (code) => {- clearTimeout(timer)- const timeoutError = timedOut- ? Object.assign(new Error('CLI command timed out'), {- code: 'ETIMEDOUT',- } as NodeJS.ErrnoException)- : undefined- resolve({- status: code,- stdout: Buffer.concat(stdoutChunks),- stderr: Buffer.concat(stderrChunks),- error: childError ?? timeoutError,- })- })-- child.stdin.write(input)- child.stdin.end()- })-}-async function assertSandboxNotListed(sandboxId: string): Promise<void> {const retries = 10const delayMs = 500for (let i = 0; i < retries; i++) {- const listResult = runCli(['sandbox', 'list', '--format', 'json'])+ const listResult = runCliInSandbox(['sandbox', 'list', '--format', 'json'])if (listResult.status === 0) {const exists = sandboxExistsInList(listResult.stdout, sandboxId)if (!exists) {@@ -273,22 +197,6 @@ function sandboxExistsInList(return parsed.some((item) => item.sandboxId === sandboxId)}-function bufferToText(value: Buffer | string | null | undefined): string {- if (!value) {- return ''- }- return typeof value === 'string' ? value : value.toString('utf8')-}--function parseEnvInt(name: string, fallback: number): number {- const raw = process.env[name]- if (!raw) {- return fallback- }- const parsed = Number.parseInt(raw, 10)- return Number.isFinite(parsed) ? parsed : fallback-}-function safeGetUserConfig(): ReturnType<typeof getUserConfig> | null {try {return getUserConfig()diff --git a/packages/cli/tests/commands/sandbox/exec_pipe.test.ts b/packages/cli/tests/commands/sandbox/exec_pipe.test.tsindex 910ce9221..d5d5bef6b 100644--- a/packages/cli/tests/commands/sandbox/exec_pipe.test.ts+++ b/packages/cli/tests/commands/sandbox/exec_pipe.test.ts@@ -1,14 +1,14 @@import { randomBytes } from 'node:crypto'-import { spawn } from 'node:child_process'-import path from 'node:path'import { describe, expect, test } from 'vitest'import { Sandbox } from 'e2b'import { getUserConfig } from 'src/user'--type UserConfigWithDomain = NonNullable<ReturnType<typeof getUserConfig>> & {- domain?: string- E2B_DOMAIN?: string-}+import {+ type CliRunResult,+ bufferToText,+ isDebug,+ parseEnvInt,+ runCliWithPipedStdin,+} from '../../setup'type PipeCase = {name: string@@ -17,11 +17,9 @@ type PipeCase = {timeoutMs?: number}-type ExecResult = {- status: number | null- stdout: Buffer- stderr: Buffer- error?: Error+type UserConfigWithDomain = NonNullable<ReturnType<typeof getUserConfig>> & {+ domain?: string+ E2B_DOMAIN?: string}const userConfig = safeGetUserConfig() as UserConfigWithDomain | null@@ -31,19 +29,15 @@ const domain =userConfig?.domain ||'e2b.app'const apiKey = process.env.E2B_API_KEY || userConfig?.teamApiKey+const shouldSkip = !apiKey || isDebug+const integrationTest = test.skipIf(shouldSkip)const templateId =process.env.E2B_PIPE_TEMPLATE_ID ||process.env.E2B_TEMPLATE_ID ||'base'-const isDebug = process.env.E2B_DEBUG !== undefined-const hasCreds = Boolean(apiKey)-const shouldSkip = !hasCreds || isDebug-const testIf = test.skipIf(shouldSkip)const includeLargeBinary =process.env.E2B_PIPE_INTEGRATION_STRICT === '1' ||process.env.E2B_PIPE_INTEGRATION_BINARY === '1' ||- process.env.E2B_PIPE_SMOKE_STRICT === '1' || // Backward compatibility.- process.env.E2B_PIPE_SMOKE_BINARY === '1' || // Backward compatibility.process.env.STRICT === '1'const sandboxTimeoutMs = parseEnvInt('E2B_PIPE_SANDBOX_TIMEOUT_MS', 10_000)const testTimeoutMs = parseEnvInt('E2B_PIPE_TEST_TIMEOUT_MS', 60_000)@@ -51,8 +45,13 @@ const defaultCmdTimeoutMs = parseEnvInt('E2B_PIPE_CMD_TIMEOUT_MS',Math.min(8_000, testTimeoutMs))+const cliEnv: NodeJS.ProcessEnv = {+ ...process.env,+ E2B_DOMAIN: domain,+ E2B_API_KEY: apiKey,+}-const cliPath = path.join(process.cwd(), 'dist', 'index.js')+delete cliEnv.E2B_DEBUGconst defaultCases: PipeCase[] = [{@@ -101,7 +100,7 @@ const largeBinaryCases: PipeCase[] = []describe('sandbox exec stdin piping (integration)', () => {- testIf(+ integrationTest('pipes stdin to remote command',{ timeout: testTimeoutMs },async () => {@@ -116,23 +115,18 @@ describe('sandbox exec stdin piping (integration)', () => {? [...defaultCases, ...largeBinaryCases]: defaultCases- const probeCase: PipeCase = {- name: 'capability_probe_ascii_newline',- data: Buffer.from('hello\n'),- expectedBytes: 6,- }- const probe = await runExecPipe(sandbox.sandboxId, probeCase)- assertExecSucceeded(probeCase.name, probe)+ // Probe with a simple case first — some environments (notably Windows+ // CI) don't expose piped stdin so the remote byte count is 0.+ const probe = cases[1] // ascii_newline+ const probeResult = await runExecPipe(sandbox.sandboxId, probe)+ assertExecSucceeded(probe.name, probeResult)- const probeStdout = bufferToText(probe.stdout).trim()+ const probeStdout = bufferToText(probeResult.stdout).trim()if (probeStdout === '0') {- // Some environments (notably Windows CI) may not expose piped stdin- // in a way that our detector treats as piped. In that case stdin isn't- // forwarded and remote byte count is 0 without the legacy warning.return}- expect(probeStdout).toBe(String(probeCase.expectedBytes))+ expect(probeStdout).toBe(String(probe.expectedBytes))for (const testCase of cases) {const result = await runExecPipe(sandbox.sandboxId, testCase)@@ -156,76 +150,20 @@ describe('sandbox exec stdin piping (integration)', () => {function runExecPipe(sandboxId: string,testCase: PipeCase-): Promise<ExecResult> {- const cliArgs = [- cliPath,- 'sandbox',- 'exec',- sandboxId,- '--',- 'sh',- '-lc',- 'wc -c',- ]-- const env: NodeJS.ProcessEnv = {- ...process.env,- E2B_DOMAIN: domain,- E2B_API_KEY: apiKey,- }- delete env.E2B_DEBUG-- return new Promise((resolve) => {- const child = spawn('node', cliArgs, {- env,- stdio: ['pipe', 'pipe', 'pipe'],- })-- const stdoutChunks: Buffer[] = []- const stderrChunks: Buffer[] = []- let childError: Error | undefined- let timedOut = false-- const timer = setTimeout(() => {- timedOut = true- child.kill()- }, testCase.timeoutMs ?? defaultCmdTimeoutMs)-- child.stdout.on('data', (chunk) => stdoutChunks.push(Buffer.from(chunk)))- child.stderr.on('data', (chunk) => stderrChunks.push(Buffer.from(chunk)))- child.on('error', (err) => {- childError = err- })- child.on('close', (code) => {- clearTimeout(timer)- const timeoutError = timedOut- ? Object.assign(new Error('CLI command timed out'), {- code: 'ETIMEDOUT',- } as NodeJS.ErrnoException)- : undefined- resolve({- status: code,- stdout: Buffer.concat(stdoutChunks),- stderr: Buffer.concat(stderrChunks),- error: childError ?? timeoutError,- })- })-- child.stdin.write(testCase.data)- child.stdin.end()- })-}--function bufferToText(value: Buffer | string | null | undefined): string {- if (!value) {- return ''- }- return typeof value === 'string' ? value : value.toString('utf8')+): Promise<CliRunResult> {+ return runCliWithPipedStdin(+ ['sandbox', 'exec', sandboxId, '--', 'sh', '-lc', 'wc -c'],+ testCase.data,+ {+ env: cliEnv,+ timeoutMs: testCase.timeoutMs ?? defaultCmdTimeoutMs,+ }+ )}function assertExecSucceeded(name: string,- result: ExecResult+ result: CliRunResult): void {if (result.error) {const timedOut = (result.error as NodeJS.ErrnoException).code === 'ETIMEDOUT'@@ -240,15 +178,6 @@ function assertExecSucceeded(}}-function parseEnvInt(name: string, fallback: number): number {- const raw = process.env[name]- if (!raw) {- return fallback- }- const parsed = Number.parseInt(raw, 10)- return Number.isFinite(parsed) ? parsed : fallback-}-function safeGetUserConfig(): ReturnType<typeof getUserConfig> | null {try {return getUserConfig()diff --git a/packages/cli/tests/setup.ts b/packages/cli/tests/setup.tsindex 9f04ae115..32c463340 100644--- a/packages/cli/tests/setup.ts+++ b/packages/cli/tests/setup.ts@@ -1,5 +1,102 @@-import { execSync } from 'node:child_process'+import { execSync, spawn, spawnSync } from 'node:child_process'+import path from 'node:path'++export const isDebug = process.env.E2B_DEBUG !== undefined++type CliRunOptions = {+ timeoutMs: number+ env?: NodeJS.ProcessEnv+}++type CliRunSyncOptions = CliRunOptions & {+ input?: string | Buffer+}++export type CliRunResult = {+ status: number | null+ stdout: Buffer+ stderr: Buffer+ error?: Error+}++const cliPath = path.join(process.cwd(), 'dist', 'index.js')export async function setup() {execSync('pnpm build', { stdio: 'inherit' })}++export function runCli(+ args: string[],+ options: CliRunSyncOptions+): ReturnType<typeof spawnSync> {+ return spawnSync('node', [cliPath, ...args], {+ env: options.env ?? process.env,+ input: options.input,+ encoding: 'utf8',+ timeout: options.timeoutMs,+ })+}++export async function runCliWithPipedStdin(+ args: string[],+ input: Buffer,+ options: CliRunOptions+): Promise<CliRunResult> {+ return await new Promise((resolve) => {+ const child = spawn('node', [cliPath, ...args], {+ env: options.env ?? process.env,+ stdio: ['pipe', 'pipe', 'pipe'],+ })++ const stdoutChunks: Buffer[] = []+ const stderrChunks: Buffer[] = []+ let childError: Error | undefined+ let timedOut = false++ const timer = setTimeout(() => {+ timedOut = true+ child.kill()+ }, options.timeoutMs)++ child.stdout.on('data', (chunk) => stdoutChunks.push(Buffer.from(chunk)))+ child.stderr.on('data', (chunk) => stderrChunks.push(Buffer.from(chunk)))+ child.on('error', (err) => {+ childError = err+ })+ child.on('close', (code) => {+ clearTimeout(timer)+ const timeoutError = timedOut+ ? Object.assign(new Error('CLI command timed out'), {+ code: 'ETIMEDOUT',+ } as NodeJS.ErrnoException)+ : undefined++ resolve({+ status: code,+ stdout: Buffer.concat(stdoutChunks),+ stderr: Buffer.concat(stderrChunks),+ error: childError ?? timeoutError,+ })+ })++ child.stdin.write(input)+ child.stdin.end()+ })+}++export function bufferToText(value: Buffer | string | null | undefined): string {+ if (!value) {+ return ''+ }+ return typeof value === 'string' ? value : value.toString('utf8')+}++export function parseEnvInt(name: string, fallback: number): number {+ const raw = process.env[name]+ if (!raw) {+ return fallback+ }++ const parsed = Number.parseInt(raw, 10)+ return Number.isFinite(parsed) ? parsed : fallback+}