Skip to content
Python

Tolerate non-UTF-8 bytes in shell command output (OPE-204)

Devika Vermaf94c2ac1b9c5
diff --git a/coworker/sandbox/runner/executor.py b/coworker/sandbox/runner/executor.py
index b6a296ad..8f5be36d 100644
--- a/coworker/sandbox/runner/executor.py
+++ b/coworker/sandbox/runner/executor.py
@@ -109,6 +109,9 @@ class _BackgroundTask:
stderr=subprocess.STDOUT,
cwd=cwd,
text=True,
+ # Never let one non-UTF-8 byte in a command's output (a binary dump, a
+ # Latin-1 log line) kill the reader thread; keep the rest of the output.
+ errors="replace",
bufsize=1,
env=env,
**spawn_kwargs,
@@ -220,6 +223,9 @@ class LocalExecutor(Executor):
stderr=subprocess.STDOUT,
cwd=self.cwd,
text=True,
+ # Never let one non-UTF-8 byte in a command's output (a binary dump, a
+ # Latin-1 log line) kill the reader thread; keep the rest of the output.
+ errors="replace",
bufsize=1,
env=self._env,
**spawn_kwargs,
diff --git a/tests/test_shell_non_utf8_output.py b/tests/test_shell_non_utf8_output.py
new file mode 100644
index 00000000..00d62705
--- /dev/null
+++ b/tests/test_shell_non_utf8_output.py
@@ -0,0 +1,54 @@
+"""A command whose output is not valid UTF-8 must not wedge the shell.
+
+The executor reads the shell's stdout in text mode. Before the fix, one undecodable byte
+(a binary dump, a Latin-1 log line) raised UnicodeDecodeError inside the reader thread,
+the thread died, and the command hung until its timeout with no output. Seen on the
+`make-mips-interpreter` task, where the agent cat'ed an ELF file.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+
+import pytest
+
+from coworker.tools.shell import LocalExecutor
+
+pytestmark = pytest.mark.skipif(
+ sys.platform == "win32", reason="POSIX shell: PowerShell re-encodes bytes before we see them"
+)
+
+# 0xE2 starts a three-byte UTF-8 sequence; followed by ASCII it is undecodable.
+BAD_BYTES = r"printf 'before\xe2after\n'"
+
+
+def test_undecodable_byte_does_not_hang_the_command(tmp_path):
+ ex = LocalExecutor(cwd=tmp_path, default_timeout=10)
+ try:
+ started = time.monotonic()
+ result = ex.run(BAD_BYTES)
+ assert time.monotonic() - started < 5, "command hung until the timeout"
+ assert result["timed_out"] is False
+ assert result["exit_code"] == 0
+ assert "before" in result["output"] and "after" in result["output"]
+ # The shell is still usable afterwards.
+ assert ex.run("echo still_alive")["output"].strip() == "still_alive"
+ finally:
+ ex.close()
+
+
+def test_undecodable_byte_in_background_output(tmp_path):
+ ex = LocalExecutor(cwd=tmp_path, default_timeout=10)
+ try:
+ task = ex.run_background(BAD_BYTES)
+ deadline = time.monotonic() + 5
+ output = ""
+ while time.monotonic() < deadline:
+ output += ex.background_output(task["task_id"]).get("output", "")
+ if "after" in output:
+ break
+ time.sleep(0.1)
+ assert "before" in output and "after" in output
+ finally:
+ ex.close()