Skip to content

The agent that turns websites into APIs!

fix(runtime): distinguish Python UTF-8 pipes from native legacy output

kalil0321dab9b3bff00e
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.py
index b77b4e8..7de2eeb 100644
--- a/src/reverse_api/cli.py
+++ b/src/reverse_api/cli.py
@@ -3095,7 +3095,8 @@ def _run_script_machine_payload(
)
return subprocess.CompletedProcess(
captured.args, captured.returncode,
- decode_process_output(captured.stdout), decode_process_output(captured.stderr),
+ decode_process_output(captured.stdout, utf8_stream=True),
+ decode_process_output(captured.stderr, utf8_stream=True),
)
result = run_client()
diff --git a/src/reverse_api/runtime_commands.py b/src/reverse_api/runtime_commands.py
index 73d0111..0840ea1 100644
--- a/src/reverse_api/runtime_commands.py
+++ b/src/reverse_api/runtime_commands.py
@@ -9,8 +9,14 @@ import sys
from pathlib import Path
-def decode_process_output(output: bytes | str | None) -> str:
- """Preserve UTF-8 tool output, with a fallback for legacy native tools."""
+def decode_process_output(output: bytes | str | None, *, utf8_stream: bool = False) -> str:
+ """Decode UTF-8 or legacy output using the caller's stream contract.
+
+ Native tools fall back as a whole to the system encoding. For Python
+ clients configured with PYTHONIOENCODING=utf-8, retain valid UTF-8 spans
+ around stray legacy bytes. Ambiguous byte sequences cannot identify
+ an arbitrary mixed encoding reliably without this caller context.
+ """
if output is None:
return ""
if isinstance(output, bytes):
@@ -20,13 +26,16 @@ def decode_process_output(output: bytes | str | None) -> str:
# Retain valid UTF-8 characters even when a native child writes
# legacy bytes into the same pipe. Keep ASCII with undecodable
# bytes: it can be the trailing byte of a legacy multibyte glyph.
- escaped = output.decode("utf-8", errors="surrogateescape")
encoding = locale.getencoding()
- text = re.sub(
- r"[\x00-\x7f\udc80-\udcff]+",
- lambda match: match.group().encode("utf-8", errors="surrogateescape").decode(encoding, errors="replace"),
- escaped,
- )
+ if not utf8_stream:
+ text = output.decode(encoding, errors="replace")
+ else:
+ escaped = output.decode("utf-8", errors="surrogateescape")
+ text = re.sub(
+ r"[\x00-\x7f\udc80-\udcff]+",
+ lambda match: match.group().encode("utf-8", errors="surrogateescape").decode(encoding, errors="replace"),
+ escaped,
+ )
else:
text = output
return text.replace("\r\n", "\n").replace("\r", "\n")
diff --git a/tests/test_runtime_commands.py b/tests/test_runtime_commands.py
index 4a4045f..969dd8a 100644
--- a/tests/test_runtime_commands.py
+++ b/tests/test_runtime_commands.py
@@ -86,4 +86,9 @@ def test_real_npx_preserves_client_arguments(tmp_path):
def test_mixed_stream_preserves_utf8_and_legacy_segments(monkeypatch, encoding, text):
monkeypatch.setattr("locale.getencoding", lambda: encoding)
data = text.encode(encoding) + " 🐍 ".encode() + text.encode(encoding)
- assert decode_process_output(data) == f"{text} 🐍 {text}"
+ assert decode_process_output(data, utf8_stream=True) == f"{text} 🐍 {text}"
+
+
+def test_native_gbk_stream_with_valid_utf8_byte_pair(monkeypatch):
+ monkeypatch.setattr("locale.getencoding", lambda: "gbk")
+ assert decode_process_output("δΈ­ζ–‡ιš†".encode("gbk")) == "δΈ­ζ–‡ιš†"