The agent that turns websites into APIs!
fix(runtime): distinguish Python UTF-8 pipes from native legacy output
kalil0321dab9b3bff00e
diff --git a/src/reverse_api/cli.py b/src/reverse_api/cli.pyindex b77b4e8..7de2eeb 100644--- a/src/reverse_api/cli.py+++ b/src/reverse_api/cli.py@@ -3095,7 +3095,8 @@ def _run_script_machine_payload()return subprocess.CompletedProcess(captured.args, captured.returncode,- decode_process_output(captured.stdout), decode_process_output(captured.stderr),+ decode_process_output(captured.stdout, utf8_stream=True),+ decode_process_output(captured.stderr, utf8_stream=True),)result = run_client()diff --git a/src/reverse_api/runtime_commands.py b/src/reverse_api/runtime_commands.pyindex 73d0111..0840ea1 100644--- a/src/reverse_api/runtime_commands.py+++ b/src/reverse_api/runtime_commands.py@@ -9,8 +9,14 @@ import sysfrom pathlib import Path-def decode_process_output(output: bytes | str | None) -> str:- """Preserve UTF-8 tool output, with a fallback for legacy native tools."""+def decode_process_output(output: bytes | str | None, *, utf8_stream: bool = False) -> str:+ """Decode UTF-8 or legacy output using the caller's stream contract.++ Native tools fall back as a whole to the system encoding. For Python+ clients configured with PYTHONIOENCODING=utf-8, retain valid UTF-8 spans+ around stray legacy bytes. Ambiguous byte sequences cannot identify+ an arbitrary mixed encoding reliably without this caller context.+ """if output is None:return ""if isinstance(output, bytes):@@ -20,13 +26,16 @@ def decode_process_output(output: bytes | str | None) -> str:# Retain valid UTF-8 characters even when a native child writes# legacy bytes into the same pipe. Keep ASCII with undecodable# bytes: it can be the trailing byte of a legacy multibyte glyph.- escaped = output.decode("utf-8", errors="surrogateescape")encoding = locale.getencoding()- text = re.sub(- r"[\x00-\x7f\udc80-\udcff]+",- lambda match: match.group().encode("utf-8", errors="surrogateescape").decode(encoding, errors="replace"),- escaped,- )+ if not utf8_stream:+ text = output.decode(encoding, errors="replace")+ else:+ escaped = output.decode("utf-8", errors="surrogateescape")+ text = re.sub(+ r"[\x00-\x7f\udc80-\udcff]+",+ lambda match: match.group().encode("utf-8", errors="surrogateescape").decode(encoding, errors="replace"),+ escaped,+ )else:text = outputreturn text.replace("\r\n", "\n").replace("\r", "\n")diff --git a/tests/test_runtime_commands.py b/tests/test_runtime_commands.pyindex 4a4045f..969dd8a 100644--- a/tests/test_runtime_commands.py+++ b/tests/test_runtime_commands.py@@ -86,4 +86,9 @@ def test_real_npx_preserves_client_arguments(tmp_path):def test_mixed_stream_preserves_utf8_and_legacy_segments(monkeypatch, encoding, text):monkeypatch.setattr("locale.getencoding", lambda: encoding)data = text.encode(encoding) + " π ".encode() + text.encode(encoding)- assert decode_process_output(data) == f"{text} π {text}"+ assert decode_process_output(data, utf8_stream=True) == f"{text} π {text}"+++def test_native_gbk_stream_with_valid_utf8_byte_pair(monkeypatch):+ monkeypatch.setattr("locale.getencoding", lambda: "gbk")+ assert decode_process_output("δΈζι".encode("gbk")) == "δΈζι"