Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
fix(sdk): correct command/PTY stream handling in Python and JS SDKs (#1441)
Mish Ushakovcb061d269b7c
diff --git a/.changeset/great-pots-deny.md b/.changeset/great-pots-deny.mdnew file mode 100644index 000000000..cc10b0366--- /dev/null+++ b/.changeset/great-pots-deny.md@@ -0,0 +1,8 @@+---+"e2b": patch+---++Fix command and PTY streaming issues:++- Decode stdout/stderr incrementally so multibyte UTF-8 characters split across command stream chunks are no longer corrupted+- Avoid mutating the caller's `envs` object when applying default `TERM`/`LANG`/`LC_ALL` values in `pty.create()`diff --git a/.changeset/heavy-pugs-smile.md b/.changeset/heavy-pugs-smile.mdnew file mode 100644index 000000000..8df3b1ddc--- /dev/null+++ b/.changeset/heavy-pugs-smile.md@@ -0,0 +1,10 @@+---+"@e2b/python-sdk": patch+---++Fix several command and PTY streaming issues:++- Decode stdout/stderr incrementally so multibyte UTF-8 characters split across stream chunks are no longer corrupted+- Return `None` instead of empty strings for unset `tag` and `cwd` fields in `commands.list()`+- Close command/PTY/watch stream connections when establishing the stream fails, instead of leaking pooled connections+- Avoid mutating the caller's `envs` dict when applying default `TERM`/`LANG`/`LC_ALL` values in `pty.create()`diff --git a/packages/js-sdk/src/sandbox/commands/commandHandle.ts b/packages/js-sdk/src/sandbox/commands/commandHandle.tsindex 0bf37bdb0..ef57b21c1 100644--- a/packages/js-sdk/src/sandbox/commands/commandHandle.ts+++ b/packages/js-sdk/src/sandbox/commands/commandHandle.ts@@ -91,6 +91,9 @@ export class CommandHandleprivate _stdout = ''private _stderr = ''+ private readonly stdoutDecoder = new TextDecoder()+ private readonly stderrDecoder = new TextDecoder()+private result?: CommandResultprivate iterationError?: Error@@ -235,41 +238,89 @@ export class CommandHandleawait this.handleCloseStdin(opts)}+ /**+ * Flush any bytes still buffered in the stream decoders.+ *+ * Incomplete trailing UTF-8 sequences are emitted as replacement+ * characters, matching the per-chunk decoding behavior.+ */+ private *flushDecoders(): Generator<+ [Stdout, null, null] | [null, Stderr, null]+ > {+ const stdoutRest = this.stdoutDecoder.decode()+ if (stdoutRest) {+ this._stdout += stdoutRest+ yield [stdoutRest as Stdout, null, null]+ }+ const stderrRest = this.stderrDecoder.decode()+ if (stderrRest) {+ this._stderr += stderrRest+ yield [null, stderrRest as Stderr, null]+ }+ }+private async *iterateEvents(): AsyncGenerator<[Stdout, null, null] | [null, Stderr, null] | [null, null, PtyOutput]> {- for await (const event of this.events) {- const e = event?.event?.event- let out: string | undefined-- switch (e?.case) {- case 'data':- switch (e.value.output.case) {- case 'stdout':- out = new TextDecoder().decode(e.value.output.value)- this._stdout += out- yield [out as Stdout, null, null]- break- case 'stderr':- out = new TextDecoder().decode(e.value.output.value)- this._stderr += out- yield [null, out as Stderr, null]- break- case 'pty':- yield [null, null, e.value.output.value as PtyOutput]- break- }- break- case 'end':- this.result = {- exitCode: e.value.exitCode,- error: e.value.error,- stdout: this.stdout,- stderr: this.stderr,+ try {+ for await (const event of this.events) {+ const e = event?.event?.event+ let out: string | undefined++ switch (e?.case) {+ case 'data':+ switch (e.value.output.case) {+ case 'stdout':+ out = this.stdoutDecoder.decode(e.value.output.value, {+ stream: true,+ })+ if (out) {+ this._stdout += out+ yield [out as Stdout, null, null]+ }+ break+ case 'stderr':+ out = this.stderrDecoder.decode(e.value.output.value, {+ stream: true,+ })+ if (out) {+ this._stderr += out+ yield [null, out as Stderr, null]+ }+ break+ case 'pty':+ yield [null, null, e.value.output.value as PtyOutput]+ break+ }+ break+ case 'end': {+ yield* this.flushDecoders()+ this.result = {+ exitCode: e.value.exitCode,+ error: e.value.error,+ stdout: this.stdout,+ stderr: this.stderr,+ }+ break}- break+ }+ // TODO: Handle empty events like in python SDK}- // TODO: Handle empty events like in python SDK+ } catch (e) {+ // The stream raised before an `end` event (e.g. disconnect or RPC+ // failure). Flush any bytes still buffered in the decoders so incomplete+ // trailing sequences surface as replacement characters instead of being+ // silently dropped, then re-raise so the error is still surfaced.+ yield* this.flushDecoders()+ throw e+ }++ // If the stream closed without an `end` event (e.g. disconnect or a+ // dropped connection), flush any bytes still buffered in the decoders so+ // incomplete trailing sequences surface as replacement characters instead+ // of being silently dropped.+ if (this.result === undefined) {+ yield* this.flushDecoders()}}diff --git a/packages/js-sdk/src/sandbox/commands/pty.ts b/packages/js-sdk/src/sandbox/commands/pty.tsindex 283190bf0..0fb685a61 100644--- a/packages/js-sdk/src/sandbox/commands/pty.ts+++ b/packages/js-sdk/src/sandbox/commands/pty.ts@@ -106,7 +106,7 @@ export class Pty {async create(opts: PtyCreateOpts) {const requestTimeoutMs =opts?.requestTimeoutMs ?? this.connectionConfig.requestTimeoutMs- const envs = opts?.envs ?? {}+ const envs = { ...(opts?.envs ?? {}) }envs.TERM = envs.TERM ?? 'xterm-256color'envs.LANG = envs.LANG ?? 'C.UTF-8'envs.LC_ALL = envs.LC_ALL ?? 'C.UTF-8'diff --git a/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts b/packages/js-sdk/tests/sandbox/commands/commandHandle.test.tsindex 71dc3a96f..65b289b60 100644--- a/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts+++ b/packages/js-sdk/tests/sandbox/commands/commandHandle.test.ts@@ -52,6 +52,36 @@ function createEvents(kind: EventKind): AsyncIterable<any> {return events()}+function dataEvent(kind: 'stdout' | 'stderr', value: Uint8Array) {+ return {+ event: {+ event: {+ case: 'data',+ value: {+ output: {+ case: kind,+ value,+ },+ },+ },+ },+ }+}++function endEvent(exitCode = 0) {+ return {+ event: {+ event: {+ case: 'end',+ value: {+ exitCode,+ error: undefined,+ },+ },+ },+ }+}+describe('CommandHandle', () => {it.each<EventKind>(['stdout', 'stderr', 'pty'])('wait awaits async %s callbacks',@@ -94,4 +124,137 @@ describe('CommandHandle', () => {expect(waitResolved).toBe(true)})++ it('decodes multibyte characters split across chunks', async () => {+ const emojiBytes = new TextEncoder().encode('😀')++ async function* events() {+ yield dataEvent(+ 'stdout',+ new Uint8Array([+ ...new TextEncoder().encode('a'),+ ...emojiBytes.slice(0, 2),+ ])+ )+ yield dataEvent(+ 'stdout',+ new Uint8Array([+ ...emojiBytes.slice(2),+ ...new TextEncoder().encode('b'),+ ])+ )+ yield dataEvent('stderr', emojiBytes.slice(0, 3))+ yield dataEvent('stderr', emojiBytes.slice(3))+ yield endEvent()+ }++ const stdoutChunks: string[] = []+ const handle = new CommandHandle(+ 1,+ () => {},+ async () => true,+ events(),+ (out) => {+ stdoutChunks.push(out)+ }+ )++ const result = await handle.wait()++ expect(result.stdout).toBe('a😀b')+ expect(result.stderr).toBe('😀')+ expect(result.stdout).not.toContain('�')+ expect(result.stderr).not.toContain('�')+ expect(stdoutChunks.join('')).toBe('a😀b')+ })++ it('replaces incomplete trailing utf-8 sequences at the end of the stream', async () => {+ const emojiBytes = new TextEncoder().encode('😀')++ async function* events() {+ yield dataEvent(+ 'stdout',+ new Uint8Array([+ ...new TextEncoder().encode('a'),+ ...emojiBytes.slice(0, 2),+ ])+ )+ yield endEvent()+ }++ const handle = new CommandHandle(+ 1,+ () => {},+ async () => true,+ events()+ )++ const result = await handle.wait()++ expect(result.stdout).toBe('a�')+ })++ it('flushes incomplete trailing utf-8 sequences when the stream closes without an end event', async () => {+ const emojiBytes = new TextEncoder().encode('😀')++ async function* events() {+ yield dataEvent(+ 'stdout',+ new Uint8Array([+ ...new TextEncoder().encode('a'),+ ...emojiBytes.slice(0, 2),+ ])+ )+ }++ const stdoutChunks: string[] = []+ const handle = new CommandHandle(+ 1,+ () => {},+ async () => true,+ events(),+ (out) => {+ stdoutChunks.push(out)+ }+ )++ // No end event arrives, so wait() rejects, but the buffered bytes must+ // still be flushed to the stdout callback as a replacement character.+ await expect(handle.wait()).rejects.toThrow()++ expect(stdoutChunks.join('')).toBe('a�')+ })++ it('flushes incomplete trailing utf-8 sequences when the stream errors', async () => {+ const emojiBytes = new TextEncoder().encode('😀')++ async function* events() {+ yield dataEvent(+ 'stdout',+ new Uint8Array([+ ...new TextEncoder().encode('a'),+ ...emojiBytes.slice(0, 2),+ ])+ )+ throw new Error('stream died')+ }++ const stdoutChunks: string[] = []+ const handle = new CommandHandle(+ 1,+ () => {},+ async () => true,+ events(),+ (out) => {+ stdoutChunks.push(out)+ }+ )++ // The stream errors before an end event arrives, so wait() rejects, but the+ // buffered bytes must still be flushed to the stdout callback as a+ // replacement character.+ await expect(handle.wait()).rejects.toThrow()++ expect(stdoutChunks.join('')).toBe('a�')+ })})diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command.py b/packages/python-sdk/e2b/sandbox_async/commands/command.pyindex 8142d5321..5bf32f616 100644--- a/packages/python-sdk/e2b/sandbox_async/commands/command.py+++ b/packages/python-sdk/e2b/sandbox_async/commands/command.py@@ -67,11 +67,11 @@ class Commands:return [ProcessInfo(pid=p.pid,- tag=p.tag,+ tag=p.tag if p.HasField("tag") else None,cmd=p.config.cmd,args=list(p.config.args),envs=dict(p.config.envs),- cwd=p.config.cwd,+ cwd=p.config.cwd if p.config.HasField("cwd") else None,)for p in res.processes]@@ -325,6 +325,10 @@ class Commands:check_health=self._check_health,)except Exception as e:+ try:+ await events.aclose()+ except Exception:+ passraise await ahandle_rpc_exception_with_health(e, self._check_health)async def connect(@@ -384,4 +388,8 @@ class Commands:check_health=self._check_health,)except Exception as e:+ try:+ await events.aclose()+ except Exception:+ passraise await ahandle_rpc_exception_with_health(e, self._check_health)diff --git a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.pyindex 7b094cd6f..bb64e41bf 100644--- a/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py+++ b/packages/python-sdk/e2b/sandbox_async/commands/command_handle.py@@ -1,10 +1,12 @@import asyncio+import codecsimport inspectfrom typing import (Optional,Callable,Any,AsyncGenerator,+ List,Awaitable,Union,Tuple,@@ -102,6 +104,9 @@ class AsyncCommandHandle:self._stdout: str = ""self._stderr: str = ""+ self._stdout_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")+ self._stderr_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")+self._on_stdout = on_stdoutself._on_stderr = on_stderrself._on_pty = on_pty@@ -111,6 +116,26 @@ class AsyncCommandHandle:self._wait = asyncio.create_task(self._handle_events())+ def _flush_decoders(+ self,+ ) -> List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]]:+ """+ Flush any bytes still buffered in the stream decoders.++ Incomplete trailing UTF-8 sequences are emitted as replacement+ characters, matching the per-chunk decoding behavior.+ """+ events: List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]] = []+ out = self._stdout_decoder.decode(b"", final=True)+ if out:+ self._stdout += out+ events.append((out, None, None))+ err = self._stderr_decoder.decode(b"", final=True)+ if err:+ self._stderr += err+ events.append((None, err, None))+ return events+async def _iterate_events(self,) -> AsyncGenerator[@@ -121,25 +146,47 @@ class AsyncCommandHandle:],None,]:- async for event in self._events:- if event.event.HasField("data"):- if event.event.data.stdout:- out = event.event.data.stdout.decode("utf-8", "replace")- self._stdout += out- yield out, None, None- if event.event.data.stderr:- out = event.event.data.stderr.decode("utf-8", "replace")- self._stderr += out- yield None, out, None- if event.event.data.pty:- yield None, None, event.event.data.pty- if event.event.HasField("end"):- self._result = CommandResult(- stdout=self._stdout,- stderr=self._stderr,- exit_code=event.event.end.exit_code,- error=event.event.end.error,- )+ try:+ async for event in self._events:+ if event.event.HasField("data"):+ if event.event.data.stdout:+ out = self._stdout_decoder.decode(event.event.data.stdout)+ if out:+ self._stdout += out+ yield out, None, None+ if event.event.data.stderr:+ out = self._stderr_decoder.decode(event.event.data.stderr)+ if out:+ self._stderr += out+ yield None, out, None+ if event.event.data.pty:+ yield None, None, event.event.data.pty+ if event.event.HasField("end"):+ for flushed in self._flush_decoders():+ yield flushed+ self._result = CommandResult(+ stdout=self._stdout,+ stderr=self._stderr,+ exit_code=event.event.end.exit_code,+ error=event.event.end.error,+ )+ except Exception:+ # The stream raised before an end event (e.g. disconnect or RPC+ # failure). Flush any bytes still buffered in the decoders so+ # incomplete trailing sequences surface as replacement characters+ # instead of being silently dropped, then re-raise so the error is+ # still surfaced by the consumer.+ for flushed in self._flush_decoders():+ yield flushed+ raise++ # If the stream closed without an end event (e.g. disconnect or a+ # dropped connection), flush any bytes still buffered in the decoders+ # so incomplete trailing sequences surface as replacement characters+ # instead of being silently dropped.+ if self._result is None:+ for flushed in self._flush_decoders():+ yield flushedasync def disconnect(self) -> None:"""diff --git a/packages/python-sdk/e2b/sandbox_async/commands/pty.py b/packages/python-sdk/e2b/sandbox_async/commands/pty.pyindex 809f061b8..a9524ec82 100644--- a/packages/python-sdk/e2b/sandbox_async/commands/pty.py+++ b/packages/python-sdk/e2b/sandbox_async/commands/pty.py@@ -129,7 +129,7 @@ class Pty::return: Handle to interact with the PTY"""- envs = envs or {}+ envs = dict(envs) if envs else {}envs.setdefault("TERM", "xterm-256color")envs.setdefault("LANG", "C.UTF-8")envs.setdefault("LC_ALL", "C.UTF-8")@@ -171,6 +171,10 @@ class Pty:check_health=self._check_health,)except Exception as e:+ try:+ await events.aclose()+ except Exception:+ passraise await ahandle_rpc_exception_with_health(e, self._check_health)async def connect(@@ -219,6 +223,10 @@ class Pty:check_health=self._check_health,)except Exception as e:+ try:+ await events.aclose()+ except Exception:+ passraise await ahandle_rpc_exception_with_health(e, self._check_health)async def resize(diff --git a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.pyindex 208a08c0f..82100d8ab 100644--- a/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py+++ b/packages/python-sdk/e2b/sandbox_async/filesystem/filesystem.py@@ -648,4 +648,8 @@ class Filesystem:check_health=lambda: acheck_sandbox_health(self._envd_api),)except Exception as e:+ try:+ await events.aclose()+ except Exception:+ passraise await _ahandle_filesystem_rpc_exception(e, self._envd_api)diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command.py b/packages/python-sdk/e2b/sandbox_sync/commands/command.pyindex 0f0585ad8..992aaf9f3 100644--- a/packages/python-sdk/e2b/sandbox_sync/commands/command.py+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command.py@@ -96,11 +96,11 @@ class Commands:return [ProcessInfo(pid=p.pid,- tag=p.tag,+ tag=p.tag if p.HasField("tag") else None,cmd=p.config.cmd,args=list(p.config.args),envs=dict(p.config.envs),- cwd=p.config.cwd,+ cwd=p.config.cwd if p.config.HasField("cwd") else None,)for p in res.processes]@@ -353,6 +353,10 @@ class Commands:check_health=self._check_health,)except Exception as e:+ try:+ events.close()+ except Exception:+ passraise handle_rpc_exception_with_health(e, self._check_health)def connect(@@ -406,4 +410,8 @@ class Commands:check_health=self._check_health,)except Exception as e:+ try:+ events.close()+ except Exception:+ passraise handle_rpc_exception_with_health(e, self._check_health)diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.pyindex 27138b9d8..ce09dffbb 100644--- a/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py+++ b/packages/python-sdk/e2b/sandbox_sync/commands/command_handle.py@@ -1,4 +1,6 @@-from typing import Optional, Callable, Any, Generator, Union, Tuple+import codecs++from typing import Optional, Callable, Any, Generator, List, Union, Tuplefrom e2b.envd.rpc import handle_rpc_exception_with_healthfrom e2b.envd.process import process_pb2@@ -49,6 +51,9 @@ class CommandHandle:self._stdout: str = ""self._stderr: str = ""+ self._stdout_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")+ self._stderr_decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")+self._result: Optional[CommandResult] = Noneself._iteration_exception: Optional[Exception] = None@@ -60,6 +65,26 @@ class CommandHandle:"""return self._handle_events()+ def _flush_decoders(+ self,+ ) -> List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]]:+ """+ Flush any bytes still buffered in the stream decoders.++ Incomplete trailing UTF-8 sequences are emitted as replacement+ characters, matching the per-chunk decoding behavior.+ """+ events: List[Union[Tuple[Stdout, None, None], Tuple[None, Stderr, None]]] = []+ out = self._stdout_decoder.decode(b"", final=True)+ if out:+ self._stdout += out+ events.append((out, None, None))+ err = self._stderr_decoder.decode(b"", final=True)+ if err:+ self._stderr += err+ events.append((None, err, None))+ return events+def _handle_events(self,) -> Generator[@@ -75,23 +100,38 @@ class CommandHandle:for event in self._events:if event.event.HasField("data"):if event.event.data.stdout:- out = event.event.data.stdout.decode("utf-8", "replace")- self._stdout += out- yield out, None, None+ out = self._stdout_decoder.decode(event.event.data.stdout)+ if out:+ self._stdout += out+ yield out, None, Noneif event.event.data.stderr:- out = event.event.data.stderr.decode("utf-8", "replace")- self._stderr += out- yield None, out, None+ out = self._stderr_decoder.decode(event.event.data.stderr)+ if out:+ self._stderr += out+ yield None, out, Noneif event.event.data.pty:yield None, None, event.event.data.ptyif event.event.HasField("end"):+ yield from self._flush_decoders()self._result = CommandResult(stdout=self._stdout,stderr=self._stderr,exit_code=event.event.end.exit_code,error=event.event.end.error,)++ # If the stream closed without an end event (e.g. disconnect or a+ # dropped connection), flush any bytes still buffered in the+ # decoders so incomplete trailing sequences surface as replacement+ # characters instead of being silently dropped.+ if self._result is None:+ yield from self._flush_decoders()except Exception as e:+ # The stream raised before an end event (e.g. disconnect or RPC+ # failure). Flush any bytes still buffered in the decoders so+ # incomplete trailing sequences surface as replacement characters+ # instead of being silently dropped, then surface the error.+ yield from self._flush_decoders()raise handle_rpc_exception_with_health(e, self._check_health)def disconnect(self) -> None:diff --git a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py b/packages/python-sdk/e2b/sandbox_sync/commands/pty.pyindex 2abf0ed85..0bcaa8990 100644--- a/packages/python-sdk/e2b/sandbox_sync/commands/pty.py+++ b/packages/python-sdk/e2b/sandbox_sync/commands/pty.py@@ -153,7 +153,7 @@ class Pty::return: Handle to interact with the PTY"""- envs = envs or {}+ envs = dict(envs) if envs else {}envs.setdefault("TERM", "xterm-256color")envs.setdefault("LANG", "C.UTF-8")envs.setdefault("LC_ALL", "C.UTF-8")@@ -194,6 +194,10 @@ class Pty:check_health=self._check_health,)except Exception as e:+ try:+ events.close()+ except Exception:+ passraise handle_rpc_exception_with_health(e, self._check_health)def connect(@@ -239,6 +243,10 @@ class Pty:check_health=self._check_health,)except Exception as e:+ try:+ events.close()+ except Exception:+ passraise handle_rpc_exception_with_health(e, self._check_health)def resize(diff --git a/packages/python-sdk/tests/test_command_handle.py b/packages/python-sdk/tests/test_command_handle.pynew file mode 100644index 000000000..7b3237c00--- /dev/null+++ b/packages/python-sdk/tests/test_command_handle.py@@ -0,0 +1,170 @@+import pytest++from e2b.envd.process import process_pb2+from e2b.sandbox_async.commands.command_handle import AsyncCommandHandle+from e2b.sandbox_sync.commands.command_handle import CommandHandle++EMOJI = "😀"+EMOJI_BYTES = EMOJI.encode("utf-8") # 4 bytes+++def _stdout_event(data: bytes) -> process_pb2.StartResponse:+ return process_pb2.StartResponse(+ event=process_pb2.ProcessEvent(+ data=process_pb2.ProcessEvent.DataEvent(stdout=data)+ )+ )+++def _stderr_event(data: bytes) -> process_pb2.StartResponse:+ return process_pb2.StartResponse(+ event=process_pb2.ProcessEvent(+ data=process_pb2.ProcessEvent.DataEvent(stderr=data)+ )+ )+++def _end_event(exit_code: int = 0) -> process_pb2.StartResponse:+ return process_pb2.StartResponse(+ event=process_pb2.ProcessEvent(+ end=process_pb2.ProcessEvent.EndEvent(+ exit_code=exit_code, exited=True, status="exited"+ )+ )+ )+++async def _kill() -> bool:+ return True+++def test_sync_decodes_multibyte_chars_split_across_chunks():+ def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])+ yield _stdout_event(EMOJI_BYTES[2:] + b"b")+ yield _stderr_event(EMOJI_BYTES[:3])+ yield _stderr_event(EMOJI_BYTES[3:])+ yield _end_event()++ chunks = []+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())+ result = handle.wait(on_stdout=chunks.append)++ assert result.stdout == f"a{EMOJI}b"+ assert result.stderr == EMOJI+ assert "�" not in result.stdout+ assert "�" not in result.stderr+ assert "".join(chunks) == f"a{EMOJI}b"+++def test_sync_replaces_incomplete_trailing_utf8():+ def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])+ yield _end_event()++ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())+ result = handle.wait()++ assert result.stdout == "a�"+++async def test_async_decodes_multibyte_chars_split_across_chunks():+ async def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])+ yield _stdout_event(EMOJI_BYTES[2:] + b"b")+ yield _stderr_event(EMOJI_BYTES[:3])+ yield _stderr_event(EMOJI_BYTES[3:])+ yield _end_event()++ chunks = []+ handle = AsyncCommandHandle(+ pid=1,+ handle_kill=_kill,+ events=events(),+ on_stdout=chunks.append,+ )+ result = await handle.wait()++ assert result.stdout == f"a{EMOJI}b"+ assert result.stderr == EMOJI+ assert "�" not in result.stdout+ assert "�" not in result.stderr+ assert "".join(chunks) == f"a{EMOJI}b"+++async def test_async_replaces_incomplete_trailing_utf8():+ async def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])+ yield _end_event()++ handle = AsyncCommandHandle(pid=1, handle_kill=_kill, events=events())+ result = await handle.wait()++ assert result.stdout == "a�"+++def test_sync_flushes_incomplete_trailing_utf8_without_end_event():+ def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])++ chunks = []+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())+ for stdout, _, _ in handle:+ if stdout is not None:+ chunks.append(stdout)++ assert "".join(chunks) == "a�"+++async def test_async_flushes_incomplete_trailing_utf8_without_end_event():+ async def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])++ chunks = []+ handle = AsyncCommandHandle(+ pid=1,+ handle_kill=_kill,+ events=events(),+ on_stdout=chunks.append,+ )+ await handle._wait++ assert "".join(chunks) == "a�"+++def test_sync_flushes_incomplete_trailing_utf8_on_stream_error():+ def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])+ raise RuntimeError("stream died")++ chunks = []+ handle = CommandHandle(pid=1, handle_kill=lambda: True, events=events())++ # The stream raises before an end event, but the buffered bytes must still+ # be flushed as a replacement character before the error is surfaced.+ with pytest.raises(RuntimeError):+ for stdout, _, _ in handle:+ if stdout is not None:+ chunks.append(stdout)++ assert "".join(chunks) == "a�"+++async def test_async_flushes_incomplete_trailing_utf8_on_stream_error():+ async def events():+ yield _stdout_event(b"a" + EMOJI_BYTES[:2])+ raise RuntimeError("stream died")++ chunks = []+ handle = AsyncCommandHandle(+ pid=1,+ handle_kill=_kill,+ events=events(),+ on_stdout=chunks.append,+ )+ await handle._wait++ # The stream raised before an end event, but the buffered bytes must still+ # be flushed to the stdout callback as a replacement character.+ assert "".join(chunks) == "a�"+ assert isinstance(handle._iteration_exception, RuntimeError)