Skip to content

The agent that turns websites into APIs!

fix(runtime): retain valid UTF-8 spans in mixed client output

kalil0321c2f37ba7f6bb
diff --git a/src/reverse_api/runtime_commands.py b/src/reverse_api/runtime_commands.py
index 9c383f5..73d0111 100644
--- a/src/reverse_api/runtime_commands.py
+++ b/src/reverse_api/runtime_commands.py
@@ -17,7 +17,16 @@ def decode_process_output(output: bytes | str | None) -> str:
try:
text = output.decode("utf-8")
except UnicodeDecodeError:
- text = output.decode(locale.getencoding(), errors="replace")
+ # Retain valid UTF-8 characters even when a native child writes
+ # legacy bytes into the same pipe. Keep ASCII with undecodable
+ # bytes: it can be the trailing byte of a legacy multibyte glyph.
+ escaped = output.decode("utf-8", errors="surrogateescape")
+ encoding = locale.getencoding()
+ text = re.sub(
+ r"[\x00-\x7f\udc80-\udcff]+",
+ lambda match: match.group().encode("utf-8", errors="surrogateescape").decode(encoding, errors="replace"),
+ escaped,
+ )
else:
text = output
return text.replace("\r\n", "\n").replace("\r", "\n")
diff --git a/tests/test_runtime_commands.py b/tests/test_runtime_commands.py
index 2653913..1cfcf71 100644
--- a/tests/test_runtime_commands.py
+++ b/tests/test_runtime_commands.py
@@ -80,3 +80,10 @@ def test_real_npx_preserves_client_arguments(tmp_path):
assert argv[0].lower().endswith("node.exe"), "must bypass the batch shim"
result = subprocess.run(argv, capture_output=True, encoding="utf-8", check=True, timeout=30)
assert json.loads(result.stdout) == arguments
+
+
[email protected]("encoding,text", [("cp1252", "café"), ("gbk", "丂中文")])
+def test_mixed_stream_preserves_utf8_and_legacy_segments(monkeypatch, encoding, text):
+ monkeypatch.setattr("locale.getencoding", lambda: encoding)
+ data = text.encode(encoding) + " 🐍 ".encode("utf-8") + text.encode(encoding)
+ assert decode_process_output(data) == f"{text} 🐍 {text}"
diff --git a/tests/test_windows_compatibility.py b/tests/test_windows_compatibility.py
index 033bae1..f528a89 100644
--- a/tests/test_windows_compatibility.py
+++ b/tests/test_windows_compatibility.py
@@ -148,7 +148,7 @@ def test_node_diagnostic_probe_decodes_utf8_under_gbk(monkeypatch):
assert "中文 — 🐍" in error
[email protected]("language", ["python", "python-legacy", "javascript"])
[email protected]("language", ["python", "python-legacy", "python-mixed", "javascript"])
def test_run_json_real_unicode_output(tmp_path, monkeypatch, language):
"""Exercise actual child pipes under a legacy locale, including a spaced path."""
import venv
@@ -169,6 +169,12 @@ def test_run_json_real_unicode_output(tmp_path, monkeypatch, language):
"sys.stderr.buffer.write('erreur café\\n'.encode('cp1252'))\n", encoding="utf-8",
)
monkeypatch.setattr("locale.getencoding", lambda: "cp1252")
+ if language == "python-mixed":
+ script.write_text(
+ "import sys\nsys.stdout.buffer.write('中文 — café 🐍'.encode('utf-8') + bytes([128]) + b'\\n')\n"
+ "sys.stderr.buffer.write('erreur — 中文'.encode('utf-8') + bytes([128]) + b'\\n')\n", encoding="utf-8",
+ )
+ monkeypatch.setattr("locale.getencoding", lambda: "cp1252")
venv.EnvBuilder(with_pip=False).create(output_dir / ".venv")
else:
script = scripts / "api_client.js"
@@ -186,8 +192,9 @@ def test_run_json_real_unicode_output(tmp_path, monkeypatch, language):
result = CliRunner().invoke(cli.main, ["run", "unicode-run", "--json"])
assert result.exit_code == 0, result.output
payload = json.loads(result.output)
- assert payload["stdout"] == ("café\n" if language == "python-legacy" else "中文 — café 🐍\n")
- assert payload["stderr"] == ("erreur café\n" if language == "python-legacy" else "erreur — 中文\n")
+ suffix = "€\n" if language == "python-mixed" else "\n"
+ assert payload["stdout"] == ("café" if language == "python-legacy" else "中文 — café 🐍") + suffix
+ assert payload["stderr"] == ("erreur café" if language == "python-legacy" else "erreur — 中文") + suffix
def test_interactive_dependency_retry_preserves_unicode(tmp_path, monkeypatch):