The agent that turns websites into APIs!
fix(runtime): retain valid UTF-8 spans in mixed client output
kalil0321c2f37ba7f6bb
diff --git a/src/reverse_api/runtime_commands.py b/src/reverse_api/runtime_commands.pyindex 9c383f5..73d0111 100644--- a/src/reverse_api/runtime_commands.py+++ b/src/reverse_api/runtime_commands.py@@ -17,7 +17,16 @@ def decode_process_output(output: bytes | str | None) -> str:try:text = output.decode("utf-8")except UnicodeDecodeError:- text = output.decode(locale.getencoding(), errors="replace")+ # Retain valid UTF-8 characters even when a native child writes+ # legacy bytes into the same pipe. Keep ASCII with undecodable+ # bytes: it can be the trailing byte of a legacy multibyte glyph.+ escaped = output.decode("utf-8", errors="surrogateescape")+ encoding = locale.getencoding()+ text = re.sub(+ r"[\x00-\x7f\udc80-\udcff]+",+ lambda match: match.group().encode("utf-8", errors="surrogateescape").decode(encoding, errors="replace"),+ escaped,+ )else:text = outputreturn text.replace("\r\n", "\n").replace("\r", "\n")diff --git a/tests/test_runtime_commands.py b/tests/test_runtime_commands.pyindex 2653913..1cfcf71 100644--- a/tests/test_runtime_commands.py+++ b/tests/test_runtime_commands.py@@ -80,3 +80,10 @@ def test_real_npx_preserves_client_arguments(tmp_path):assert argv[0].lower().endswith("node.exe"), "must bypass the batch shim"result = subprocess.run(argv, capture_output=True, encoding="utf-8", check=True, timeout=30)assert json.loads(result.stdout) == arguments++[email protected]("encoding,text", [("cp1252", "café"), ("gbk", "丂中文")])+def test_mixed_stream_preserves_utf8_and_legacy_segments(monkeypatch, encoding, text):+ monkeypatch.setattr("locale.getencoding", lambda: encoding)+ data = text.encode(encoding) + " 🐍 ".encode("utf-8") + text.encode(encoding)+ assert decode_process_output(data) == f"{text} 🐍 {text}"diff --git a/tests/test_windows_compatibility.py b/tests/test_windows_compatibility.pyindex 033bae1..f528a89 100644--- a/tests/test_windows_compatibility.py+++ b/tests/test_windows_compatibility.py@@ -148,7 +148,7 @@ def test_node_diagnostic_probe_decodes_utf8_under_gbk(monkeypatch):assert "中文 — 🐍" in error[email protected]("language", ["python", "python-legacy", "javascript"])[email protected]("language", ["python", "python-legacy", "python-mixed", "javascript"])def test_run_json_real_unicode_output(tmp_path, monkeypatch, language):"""Exercise actual child pipes under a legacy locale, including a spaced path."""import venv@@ -169,6 +169,12 @@ def test_run_json_real_unicode_output(tmp_path, monkeypatch, language):"sys.stderr.buffer.write('erreur café\\n'.encode('cp1252'))\n", encoding="utf-8",)monkeypatch.setattr("locale.getencoding", lambda: "cp1252")+ if language == "python-mixed":+ script.write_text(+ "import sys\nsys.stdout.buffer.write('中文 — café 🐍'.encode('utf-8') + bytes([128]) + b'\\n')\n"+ "sys.stderr.buffer.write('erreur — 中文'.encode('utf-8') + bytes([128]) + b'\\n')\n", encoding="utf-8",+ )+ monkeypatch.setattr("locale.getencoding", lambda: "cp1252")venv.EnvBuilder(with_pip=False).create(output_dir / ".venv")else:script = scripts / "api_client.js"@@ -186,8 +192,9 @@ def test_run_json_real_unicode_output(tmp_path, monkeypatch, language):result = CliRunner().invoke(cli.main, ["run", "unicode-run", "--json"])assert result.exit_code == 0, result.outputpayload = json.loads(result.output)- assert payload["stdout"] == ("café\n" if language == "python-legacy" else "中文 — café 🐍\n")- assert payload["stderr"] == ("erreur café\n" if language == "python-legacy" else "erreur — 中文\n")+ suffix = "€\n" if language == "python-mixed" else "\n"+ assert payload["stdout"] == ("café" if language == "python-legacy" else "中文 — café 🐍") + suffix+ assert payload["stderr"] == ("erreur café" if language == "python-legacy" else "erreur — 中文") + suffixdef test_interactive_dependency_retry_preserves_unicode(tmp_path, monkeypatch):