Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

fix(sdk): correct command/PTY stream handling in Python and JS SDKs (#1441)

Mish Ushakovcb061d269b7c
diff --git a/.changeset/great-pots-deny.md b/.changeset/great-pots-deny.md
new file mode 100644
index 000000000..cc10b0366
--- /dev/null
+++ b/.changeset/great-pots-deny.md
@@ -0,0 +1,8 @@
+---
+"e2b": patch
+---
+
+Fix command and PTY streaming issues:
+
+- Decode stdout/stderr incrementally so multibyte UTF-8 characters split across command stream chunks are no longer corrupted
+- Avoid mutating the caller's `envs` object when applying default `TERM`/`LANG`/`LC_ALL` values in `pty.create()`
diff --git a/.changeset/heavy-pugs-smile.md b/.changeset/heavy-pugs-smile.md
new file mode 100644
index 000000000..8df3b1ddc
--- /dev/null
+++ b/.changeset/heavy-pugs-smile.md
@@ -0,0 +1,10 @@
+---
+"@e2b/python-sdk": patch
+---
+
+Fix several command and PTY streaming issues:
+
+- Decode stdout/stderr incrementally so multibyte UTF-8 characters split across stream chunks are no longer corrupted
+- Return `None` instead of empty strings for unset `tag` and `cwd` fields in `commands.list()`
+- Close command/PTY/watch stream connections when establishing the stream fails, instead of leaking pooled connections
+- Avoid mutating the caller's `envs` dict when applying default `TERM`/`LANG`/`LC_ALL` values in `pty.create()`
diff --git a/packages/js-sdk/src/sandbox/commands/commandHandle.ts b/packages/js-sdk/src/sandbox/commands/commandHandle.ts
index 0bf37bdb0..ef57b21c1 100644
--- a/packages/js-sdk/src/sandbox/commands/commandHandle.ts
+++ b/packages/js-sdk/src/sandbox/commands/commandHandle.ts
@@ -91,6 +91,9 @@ export class CommandHandle
private _stdout = ''
private _stderr = ''
+ private readonly stdoutDecoder = new TextDecoder()
+ private readonly stderrDecoder = new TextDecoder()
+
private result?: CommandResult
private iterationError?: Error
@@ -235,41 +238,89 @@ export class CommandHandle
await this.handleCloseStdin(opts)
}
+ /**
+ * Flush any bytes still buffered in the stream decoders.
+ *
+ * Incomplete trailing UTF-8 sequences are emitted as replacement
+ * characters, matching the per-chunk decoding behavior.
+ */
+ private *flushDecoders(): Generator<
+ [Stdout, null, null] | [null, Stderr, null]
+ > {
+ const stdoutRest = this.stdoutDecoder.decode()
+ if (stdoutRest) {
+ this._stdout += stdoutRest
+ yield [stdoutRest as Stdout, null, null]
+ }
+ const stderrRest = this.stderrDecoder.decode()
+ if (stderrRest) {
+ this._stderr += stderrRest
+ yield [null, stderrRest as Stderr, null]
+ }
+ }
+
private async *iterateEvents(): AsyncGenerator<
[Stdout, null, null] | [null, Stderr, null] | [null, null, PtyOutput]
> {
- for await (const event of this.events) {
- const e = event?.event?.event
- let out: string | undefined
-
- switch (e?.case) {
- case 'data':
- switch (e.value.output.case) {
- case 'stdout':
- out = new TextDecoder().decode(e.value.output.value)
- this._stdout += out
- yield [out as Stdout, null, null]
- break
- case 'stderr':
- out = new TextDecoder().decode(e.value.output.value)
- this._stderr += out
- yield [null, out as Stderr, null]
- break
- case 'pty':
- yield [null, null, e.value.output.value as PtyOutput]
- break
- }
- break
- case 'end':
- this.result = {
- exitCode: e.value.exitCode,
- error: e.value.error,
- stdout: this.stdout,
- stderr: this.stderr,
+ try {
+ for await (const event of this.events) {
+ const e = event?.event?.event
+ let out: string | undefined
+
+ switch (e?.case) {
+ case 'data':
+ switch (e.value.output.case) {
+ case 'stdout':
+ out = this.stdoutDecoder.decode(e.value.output.value, {
+ stream: true,
+ })
+ if (out) {
+ this._stdout += out
+ yield [out as Stdout, null, null]
+ }
+ break
+ case 'stderr':
+ out = this.stderrDecoder.decode(e.value.output.value, {
+ stream: true,
+ })
+ if (out) {
+ this._stderr += out
+ yield [null, out as Stderr, null]
+ }
+ break
+ case 'pty':
+ yield [null, null, e.value.output.value as PtyOutput]
+ break
+ }
+ break
+ case 'end': {
+ yield* this.flushDecoders()
+ this.result = {
+ exitCode: e.value.exitCode,
+ error: e.value.error,
+ stdout: this.stdout,
+ stderr: this.stderr,
+ }
+ break
}
- break
+ }
+ // TODO: Handle empty events like in python SDK
}
- // TODO: Handle empty events like in python SDK
+ } catch (e) {
+ // The stream raised before an `end` event (e.g. disconnect or RPC
+ // failure). Flush any bytes still buffered in the decoders so incomplete
+ // trailing sequences surface as replacement characters instead of being
+ // silently dropped, then re-raise so the error is still surfaced.
+ yield* this.flushDecoders()
+ throw e
+ }
+
+ // If the stream closed without an `end` event (e.g. disconnect or a
+ // dropped connection), flush any bytes still buffered in the decoders so
+ // incomplete trailing sequences surface as replacement characters instead
+ // of being silently dropped.
+ if (this.result === undefined) {
+ yield* this.flushDecoders()
}
}
diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.ts
index 283190bf0..0fb685a61 100644
--- a/packages/js-sdk/src/sandbox/commands/pty.ts
+++ b/packages/js-sdk/src/sandbox/commands/pty.ts
@@ -106,7 +106,7 @@ export class Pty {
async create(opts: PtyCreateOpts) {
const requestTimeoutMs =
opts?.requestTimeoutMs ?? this.connectionConfig.requestTimeoutMs
- const envs = opts?.envs ?? {}
+ const envs = { ...(opts?.envs ?? {}) }
envs.TERM = envs.TERM ?? 'xterm-256color'
envs.LANG = envs.LANG ?? 'C.UTF-8'
envs.LC_ALL = envs.LC_ALL ?? 'C.UTF-8'
diff --git a/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts b/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts
index 71dc3a96f..65b289b60 100644
--- a/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts
+++ b/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts
@@ -52,6 +52,36 @@ function createEvents(kind: EventKind): AsyncIterable<any> {
return events()
}
+function dataEvent(kind: 'stdout' | 'stderr', value: Uint8Array) {
+ return {
+ event: {
+ event: {
+ case: 'data',
+ value: {
+ output: {
+ case: kind,
+ value,
+ },
+ },
+ },
+ },
+ }
+}
+
+function endEvent(exitCode = 0) {
+ return {
+ event: {
+ event: {
+ case: 'end',
+ value: {
+ exitCode,
+ error: undefined,
+ },
+ },
+ },
+ }
+}
+
describe('CommandHandle', () => {
it.each<EventKind>(['stdout', 'stderr', 'pty'])(
'wait awaits async %s callbacks',
@@ -94,4 +124,137 @@ describe('CommandHandle', () => {
expect(waitResolved).toBe(true)
}
)
+
+ it('decodes multibyte characters split across chunks', async () => {
+ const emojiBytes = new TextEncoder().encode('😀')
+
+ async function* events() {
+ yield dataEvent(
+ 'stdout',
+ new Uint8Array([
+ ...new TextEncoder().encode('a'),
+ ...emojiBytes.slice(0, 2),
+ ])
+ )
+ yield dataEvent(
+ 'stdout',
+ new Uint8Array([
+ ...emojiBytes.slice(2),
+ ...new TextEncoder().encode('b'),
+ ])
+ )
+ yield dataEvent('stderr', emojiBytes.slice(0, 3))
+ yield dataEvent('stderr', emojiBytes.slice(3))
+ yield endEvent()
+ }
+
+ const stdoutChunks: string[] = []
+ const handle = new CommandHandle(
+ 1,
+ () => {},
+ async () => true,
+ events(),
+ (out) => {
+ stdoutChunks.push(out)
+ }
+ )
+
+ const result = await handle.wait()
+
+ expect(result.stdout).toBe('a😀b')
+ expect(result.stderr).toBe('😀')
+ expect(result.stdout).not.toContain('�')
+ expect(result.stderr).not.toContain('�')
+ expect(stdoutChunks.join('')).toBe('a😀b')
+ })
+
+ it('replaces incomplete trailing utf-8 sequences at the end of the stream', async () => {
+ const emojiBytes = new TextEncoder().encode('😀')
+
+ async function* events() {
+ yield dataEvent(
+ 'stdout',
+ new Uint8Array([
+ ...new TextEncoder().encode('a'),
+ ...emojiBytes.slice(0, 2),
+ ])
+ )
+ yield endEvent()
+ }
+
+ const handle = new CommandHandle(
+ 1,
+ () => {},
+ async () => true,
+ events()
+ )
+
+ const result = await handle.wait()
+
+ expect(result.stdout).toBe('a�')
+ })
+
+ it('flushes incomplete trailing utf-8 sequences when the stream closes without an end event', async () => {
+ const emojiBytes = new TextEncoder().encode('😀')
+
+ async function* events() {
+ yield dataEvent(
+ 'stdout',
+ new Uint8Array([
+ ...new TextEncoder().encode('a'),
+ ...emojiBytes.slice(0, 2),
+ ])
+ )
+ }
+
+ const stdoutChunks: string[] = []
+ const handle = new CommandHandle(
+ 1,
+ () => {},
+ async () => true,
+ events(),
+ (out) => {
+ stdoutChunks.push(out)
+ }
+ )
+
+ // No end event arrives, so wait() rejects, but the buffered bytes must
+ // still be flushed to the stdout callback as a replacement character.
+ await expect(handle.wait()).rejects.toThrow()
+
+ expect(stdoutChunks.join('')).toBe('a�')
+ })
+
+ it('flushes incomplete trailing utf-8 sequences when the stream errors', async () => {
+ const emojiBytes = new TextEncoder().encode('😀')
+
+ async function* events() {
+ yield dataEvent(
+ 'stdout',
+ new Uint8Array([
+ ...new TextEncoder().encode('a'),
+ ...emojiBytes.slice(0, 2),
+ ])
+ )
+ throw new Error('stream died')
+ }
+
+ const stdoutChunks: string[] = []
+ const handle = new CommandHandle(
+ 1,
+ () => {},
+ async () => true,
+ events(),
+ (out) => {
+ stdoutChunks.push(out)
+ }
+ )
+
+ // The stream errors before an end event arrives, so wait() rejects, but the
+ // buffered bytes must still be flushed to the stdout callback as a
+ // replacement character.
+ await expect(handle.wait()).rejects.toThrow()
+
+ expect(stdoutChunks.join('')).toBe('a�')
+ })
})
diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.py
index 8142d5321..5bf32f616 100644
--- a/packages/python-sdk/e2b/sandbox_async/commands/command.py
+++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py
@@ -67,11 +67,11 @@ class Commands:
return [
ProcessInfo(
pid=p.pid,
- tag=p.tag,
+ tag=p.tag if p.HasField("tag") else None,
cmd=p.config.cmd,
args=list(p.config.args),
envs=dict(p.config.envs),
- cwd=p.config.cwd,
+ cwd=p.config.cwd if p.config.HasField("cwd") else None,
)
for p in res.processes
]
@@ -325,6 +325,10 @@ class Commands:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ await events.aclose()
+ except Exception:
+ pass
raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def connect(
@@ -384,4 +388,8 @@ class Commands:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ await events.aclose()
+ except Exception:
+ pass
raise await ahandle_rpc_exception_with_health(e, self._check_health)
diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py
index 7b094cd6f..bb64e41bf 100644
--- a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py
+++ b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py
@@ -1,10 +1,12 @@
import asyncio
+import codecs
import inspect
from typing import (
Optional,
Callable,
Any,
AsyncGenerator,
+ List,
Awaitable,
Union,
Tuple,
@@ -102,6 +104,9 @@ class AsyncCommandHandle:
self._stdout: str = ""
self._stderr: str = ""
+ self._stdout_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+ self._stderr_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+
self._on_stdout = on_stdout
self._on_stderr = on_stderr
self._on_pty = on_pty
@@ -111,6 +116,26 @@ class AsyncCommandHandle:
self._wait = asyncio.create_task(self._handle_events())
+ def _flush_decoders(
+ self,
+ ) -> List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]]:
+ """
+ Flush any bytes still buffered in the stream decoders.
+
+ Incomplete trailing UTF-8 sequences are emitted as replacement
+ characters, matching the per-chunk decoding behavior.
+ """
+ events: List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]] = []
+ out = self._stdout_decoder.decode(b"", final=True)
+ if out:
+ self._stdout += out
+ events.append((out, None, None))
+ err = self._stderr_decoder.decode(b"", final=True)
+ if err:
+ self._stderr += err
+ events.append((None, err, None))
+ return events
+
async def _iterate_events(
self,
) -> AsyncGenerator[
@@ -121,25 +146,47 @@ class AsyncCommandHandle:
],
None,
]:
- async for event in self._events:
- if event.event.HasField("data"):
- if event.event.data.stdout:
- out = event.event.data.stdout.decode("utf-8", "replace")
- self._stdout += out
- yield out, None, None
- if event.event.data.stderr:
- out = event.event.data.stderr.decode("utf-8", "replace")
- self._stderr += out
- yield None, out, None
- if event.event.data.pty:
- yield None, None, event.event.data.pty
- if event.event.HasField("end"):
- self._result = CommandResult(
- stdout=self._stdout,
- stderr=self._stderr,
- exit_code=event.event.end.exit_code,
- error=event.event.end.error,
- )
+ try:
+ async for event in self._events:
+ if event.event.HasField("data"):
+ if event.event.data.stdout:
+ out = self._stdout_decoder.decode(event.event.data.stdout)
+ if out:
+ self._stdout += out
+ yield out, None, None
+ if event.event.data.stderr:
+ out = self._stderr_decoder.decode(event.event.data.stderr)
+ if out:
+ self._stderr += out
+ yield None, out, None
+ if event.event.data.pty:
+ yield None, None, event.event.data.pty
+ if event.event.HasField("end"):
+ for flushed in self._flush_decoders():
+ yield flushed
+ self._result = CommandResult(
+ stdout=self._stdout,
+ stderr=self._stderr,
+ exit_code=event.event.end.exit_code,
+ error=event.event.end.error,
+ )
+ except Exception:
+ # The stream raised before an end event (e.g. disconnect or RPC
+ # failure). Flush any bytes still buffered in the decoders so
+ # incomplete trailing sequences surface as replacement characters
+ # instead of being silently dropped, then re-raise so the error is
+ # still surfaced by the consumer.
+ for flushed in self._flush_decoders():
+ yield flushed
+ raise
+
+ # If the stream closed without an end event (e.g. disconnect or a
+ # dropped connection), flush any bytes still buffered in the decoders
+ # so incomplete trailing sequences surface as replacement characters
+ # instead of being silently dropped.
+ if self._result is None:
+ for flushed in self._flush_decoders():
+ yield flushed
async def disconnect(self) -> None:
"""
diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.py
index 809f061b8..a9524ec82 100644
--- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py
+++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py
@@ -129,7 +129,7 @@ class Pty:
:return: Handle to interact with the PTY
"""
- envs = envs or {}
+ envs = dict(envs) if envs else {}
envs.setdefault("TERM", "xterm-256color")
envs.setdefault("LANG", "C.UTF-8")
envs.setdefault("LC_ALL", "C.UTF-8")
@@ -171,6 +171,10 @@ class Pty:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ await events.aclose()
+ except Exception:
+ pass
raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def connect(
@@ -219,6 +223,10 @@ class Pty:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ await events.aclose()
+ except Exception:
+ pass
raise await ahandle_rpc_exception_with_health(e, self._check_health)
async def resize(
diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
index 208a08c0f..82100d8ab 100644
--- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py
@@ -648,4 +648,8 @@ class Filesystem:
check_health=lambda: acheck_sandbox_health(self._envd_api),
)
except Exception as e:
+ try:
+ await events.aclose()
+ except Exception:
+ pass
raise await _ahandle_filesystem_rpc_exception(e, self._envd_api)
diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.py
index 0f0585ad8..992aaf9f3 100644
--- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py
+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py
@@ -96,11 +96,11 @@ class Commands:
return [
ProcessInfo(
pid=p.pid,
- tag=p.tag,
+ tag=p.tag if p.HasField("tag") else None,
cmd=p.config.cmd,
args=list(p.config.args),
envs=dict(p.config.envs),
- cwd=p.config.cwd,
+ cwd=p.config.cwd if p.config.HasField("cwd") else None,
)
for p in res.processes
]
@@ -353,6 +353,10 @@ class Commands:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ events.close()
+ except Exception:
+ pass
raise handle_rpc_exception_with_health(e, self._check_health)
def connect(
@@ -406,4 +410,8 @@ class Commands:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ events.close()
+ except Exception:
+ pass
raise handle_rpc_exception_with_health(e, self._check_health)
diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py
index 27138b9d8..ce09dffbb 100644
--- a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py
+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py
@@ -1,4 +1,6 @@
-from typing import Optional, Callable, Any, Generator, Union, Tuple
+import codecs
+
+from typing import Optional, Callable, Any, Generator, List, Union, Tuple
from e2b.envd.rpc import handle_rpc_exception_with_health
from e2b.envd.process import process_pb2
@@ -49,6 +51,9 @@ class CommandHandle:
self._stdout: str = ""
self._stderr: str = ""
+ self._stdout_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+ self._stderr_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
+
self._result: Optional[CommandResult] = None
self._iteration_exception: Optional[Exception] = None
@@ -60,6 +65,26 @@ class CommandHandle:
"""
return self._handle_events()
+ def _flush_decoders(
+ self,
+ ) -> List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]]:
+ """
+ Flush any bytes still buffered in the stream decoders.
+
+ Incomplete trailing UTF-8 sequences are emitted as replacement
+ characters, matching the per-chunk decoding behavior.
+ """
+ events: List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]] = []
+ out = self._stdout_decoder.decode(b"", final=True)
+ if out:
+ self._stdout += out
+ events.append((out, None, None))
+ err = self._stderr_decoder.decode(b"", final=True)
+ if err:
+ self._stderr += err
+ events.append((None, err, None))
+ return events
+
def _handle_events(
self,
) -> Generator[
@@ -75,23 +100,38 @@ class CommandHandle:
for event in self._events:
if event.event.HasField("data"):
if event.event.data.stdout:
- out = event.event.data.stdout.decode("utf-8", "replace")
- self._stdout += out
- yield out, None, None
+ out = self._stdout_decoder.decode(event.event.data.stdout)
+ if out:
+ self._stdout += out
+ yield out, None, None
if event.event.data.stderr:
- out = event.event.data.stderr.decode("utf-8", "replace")
- self._stderr += out
- yield None, out, None
+ out = self._stderr_decoder.decode(event.event.data.stderr)
+ if out:
+ self._stderr += out
+ yield None, out, None
if event.event.data.pty:
yield None, None, event.event.data.pty
if event.event.HasField("end"):
+ yield from self._flush_decoders()
self._result = CommandResult(
stdout=self._stdout,
stderr=self._stderr,
exit_code=event.event.end.exit_code,
error=event.event.end.error,
)
+
+ # If the stream closed without an end event (e.g. disconnect or a
+ # dropped connection), flush any bytes still buffered in the
+ # decoders so incomplete trailing sequences surface as replacement
+ # characters instead of being silently dropped.
+ if self._result is None:
+ yield from self._flush_decoders()
except Exception as e:
+ # The stream raised before an end event (e.g. disconnect or RPC
+ # failure). Flush any bytes still buffered in the decoders so
+ # incomplete trailing sequences surface as replacement characters
+ # instead of being silently dropped, then surface the error.
+ yield from self._flush_decoders()
raise handle_rpc_exception_with_health(e, self._check_health)
def disconnect(self) -> None:
diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py
index 2abf0ed85..0bcaa8990 100644
--- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py
+++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py
@@ -153,7 +153,7 @@ class Pty:
:return: Handle to interact with the PTY
"""
- envs = envs or {}
+ envs = dict(envs) if envs else {}
envs.setdefault("TERM", "xterm-256color")
envs.setdefault("LANG", "C.UTF-8")
envs.setdefault("LC_ALL", "C.UTF-8")
@@ -194,6 +194,10 @@ class Pty:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ events.close()
+ except Exception:
+ pass
raise handle_rpc_exception_with_health(e, self._check_health)
def connect(
@@ -239,6 +243,10 @@ class Pty:
check_health=self._check_health,
)
except Exception as e:
+ try:
+ events.close()
+ except Exception:
+ pass
raise handle_rpc_exception_with_health(e, self._check_health)
def resize(
diff --git a/packages/python-sdk/tests/test_command_handle.py b/packages/python-sdk/tests/test_command_handle.py
new file mode 100644
index 000000000..7b3237c00
--- /dev/null
+++ b/packages/python-sdk/tests/test_command_handle.py
@@ -0,0 +1,170 @@
+import pytest
+
+from e2b.envd.process import process_pb2
+from e2b.sandbox_async.commands.command_handle import AsyncCommandHandle
+from e2b.sandbox_sync.commands.command_handle import CommandHandle
+
+EMOJI = "😀"
+EMOJI_BYTES = EMOJI.encode("utf-8") # 4 bytes
+
+
+def _stdout_event(data: bytes) -> process_pb2.StartResponse:
+ return process_pb2.StartResponse(
+ event=process_pb2.ProcessEvent(
+ data=process_pb2.ProcessEvent.DataEvent(stdout=data)
+ )
+ )
+
+
+def _stderr_event(data: bytes) -> process_pb2.StartResponse:
+ return process_pb2.StartResponse(
+ event=process_pb2.ProcessEvent(
+ data=process_pb2.ProcessEvent.DataEvent(stderr=data)
+ )
+ )
+
+
+def _end_event(exit_code: int = 0) -> process_pb2.StartResponse:
+ return process_pb2.StartResponse(
+ event=process_pb2.ProcessEvent(
+ end=process_pb2.ProcessEvent.EndEvent(
+ exit_code=exit_code, exited=True, status="exited"
+ )
+ )
+ )
+
+
+async def _kill() -> bool:
+ return True
+
+
+def test_sync_decodes_multibyte_chars_split_across_chunks():
+ def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+ yield _stdout_event(EMOJI_BYTES[2:] + b"b")
+ yield _stderr_event(EMOJI_BYTES[:3])
+ yield _stderr_event(EMOJI_BYTES[3:])
+ yield _end_event()
+
+ chunks = []
+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())
+ result = handle.wait(on_stdout=chunks.append)
+
+ assert result.stdout == f"a{EMOJI}b"
+ assert result.stderr == EMOJI
+ assert "�" not in result.stdout
+ assert "�" not in result.stderr
+ assert "".join(chunks) == f"a{EMOJI}b"
+
+
+def test_sync_replaces_incomplete_trailing_utf8():
+ def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+ yield _end_event()
+
+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())
+ result = handle.wait()
+
+ assert result.stdout == "a�"
+
+
+async def test_async_decodes_multibyte_chars_split_across_chunks():
+ async def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+ yield _stdout_event(EMOJI_BYTES[2:] + b"b")
+ yield _stderr_event(EMOJI_BYTES[:3])
+ yield _stderr_event(EMOJI_BYTES[3:])
+ yield _end_event()
+
+ chunks = []
+ handle = AsyncCommandHandle(
+ pid=1,
+ handle_kill=_kill,
+ events=events(),
+ on_stdout=chunks.append,
+ )
+ result = await handle.wait()
+
+ assert result.stdout == f"a{EMOJI}b"
+ assert result.stderr == EMOJI
+ assert "�" not in result.stdout
+ assert "�" not in result.stderr
+ assert "".join(chunks) == f"a{EMOJI}b"
+
+
+async def test_async_replaces_incomplete_trailing_utf8():
+ async def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+ yield _end_event()
+
+ handle = AsyncCommandHandle(pid=1, handle_kill=_kill, events=events())
+ result = await handle.wait()
+
+ assert result.stdout == "a�"
+
+
+def test_sync_flushes_incomplete_trailing_utf8_without_end_event():
+ def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+
+ chunks = []
+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())
+ for stdout, _, _ in handle:
+ if stdout is not None:
+ chunks.append(stdout)
+
+ assert "".join(chunks) == "a�"
+
+
+async def test_async_flushes_incomplete_trailing_utf8_without_end_event():
+ async def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+
+ chunks = []
+ handle = AsyncCommandHandle(
+ pid=1,
+ handle_kill=_kill,
+ events=events(),
+ on_stdout=chunks.append,
+ )
+ await handle._wait
+
+ assert "".join(chunks) == "a�"
+
+
+def test_sync_flushes_incomplete_trailing_utf8_on_stream_error():
+ def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+ raise RuntimeError("stream died")
+
+ chunks = []
+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())
+
+ # The stream raises before an end event, but the buffered bytes must still
+ # be flushed as a replacement character before the error is surfaced.
+ with pytest.raises(RuntimeError):
+ for stdout, _, _ in handle:
+ if stdout is not None:
+ chunks.append(stdout)
+
+ assert "".join(chunks) == "a�"
+
+
+async def test_async_flushes_incomplete_trailing_utf8_on_stream_error():
+ async def events():
+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])
+ raise RuntimeError("stream died")
+
+ chunks = []
+ handle = AsyncCommandHandle(
+ pid=1,
+ handle_kill=_kill,
+ events=events(),
+ on_stdout=chunks.append,
+ )
+ await handle._wait
+
+ # The stream raised before an end event, but the buffered bytes must still
+ # be flushed to the stdout callback as a replacement character.
+ assert "".join(chunks) == "a�"
+ assert isinstance(handle._iteration_exception, RuntimeError)