Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.
Python agent-infrastructureai-agentai-searchautomationbilibiliclaude-codeclicursorfree-apillm-toolsmcppythonreddit-scrapertwitter-scraperweb-scraperxiaohongshuyoutube-transcript
fix(windows): force UTF-8 in channel subprocesses and MCP server env (#345)
Pnantb0e010c6ccb4
diff --git a/agent_reach/channels/douyin.py b/agent_reach/channels/douyin.pyindex b693dbf4..08421b04 100644--- a/agent_reach/channels/douyin.py+++ b/agent_reach/channels/douyin.py@@ -3,6 +3,9 @@import shutilimport subprocess++from agent_reach.utils.process import utf8_subprocess_env+from .base import Channel@@ -31,7 +34,8 @@ class DouyinChannel(Channel):try:r = subprocess.run([mcporter, "config", "list"], capture_output=True,- encoding="utf-8", errors="replace", timeout=5+ encoding="utf-8", errors="replace", timeout=5,+ env=utf8_subprocess_env(),)if "douyin" not in r.stdout:return "off", (@@ -47,7 +51,8 @@ class DouyinChannel(Channel):try:r = subprocess.run([mcporter, "list", "douyin"],- capture_output=True, encoding="utf-8", errors="replace", timeout=15+ capture_output=True, encoding="utf-8", errors="replace", timeout=15,+ env=utf8_subprocess_env(),)if r.returncode == 0 and r.stdout.strip():return "ok", "完整可用(视频解析、下载链接获取)"diff --git a/agent_reach/channels/linkedin.py b/agent_reach/channels/linkedin.pyindex 4fc6481c..1c95dc86 100644--- a/agent_reach/channels/linkedin.py+++ b/agent_reach/channels/linkedin.py@@ -3,6 +3,9 @@import shutilimport subprocess++from agent_reach.utils.process import utf8_subprocess_env+from .base import Channel@@ -28,7 +31,8 @@ class LinkedInChannel(Channel):try:r = subprocess.run([mcporter, "config", "list"], capture_output=True,- encoding="utf-8", errors="replace", timeout=5+ encoding="utf-8", errors="replace", timeout=5,+ env=utf8_subprocess_env(),)if "linkedin" in r.stdout.lower():return "ok", "完整可用(Profile、公司、职位搜索)"diff --git a/agent_reach/channels/weibo.py b/agent_reach/channels/weibo.pyindex cf2fc025..17563e2b 100644--- a/agent_reach/channels/weibo.py+++ b/agent_reach/channels/weibo.py@@ -3,6 +3,9 @@import shutilimport subprocess++from agent_reach.utils.process import utf8_subprocess_env+from .base import Channel@@ -24,26 +27,30 @@ class WeiboChannel(Channel):"需要 mcporter + mcp-server-weibo。安装步骤:\n"" 1. npm install -g mcporter\n"" 2. pip install git+https://github.com/Panniantong/mcp-server-weibo.git\n"- " 3. mcporter config add weibo --command 'mcp-server-weibo'\n"+ " 3. mcporter config add weibo --command 'mcp-server-weibo' "+ "--env PYTHONUTF8=1 --env PYTHONIOENCODING=utf-8\n"" 详见 https://github.com/Panniantong/mcp-server-weibo")try:r = subprocess.run([mcporter, "config", "list"], capture_output=True,- encoding="utf-8", errors="replace", timeout=5+ encoding="utf-8", errors="replace", timeout=5,+ env=utf8_subprocess_env(),)if "weibo" not in r.stdout:return "off", ("mcporter 已装但微博 MCP 未配置。运行:\n"" pip install git+https://github.com/Panniantong/mcp-server-weibo.git\n"- " mcporter config add weibo --command 'mcp-server-weibo'"+ " mcporter config add weibo --command 'mcp-server-weibo' "+ "--env PYTHONUTF8=1 --env PYTHONIOENCODING=utf-8")except Exception:return "off", "mcporter 连接异常"try:r = subprocess.run([mcporter, "list", "weibo"], capture_output=True,- encoding="utf-8", errors="replace", timeout=15+ encoding="utf-8", errors="replace", timeout=15,+ env=utf8_subprocess_env(),)if r.returncode == 0 and "search_users" in r.stdout:return "ok", "完整可用(热搜、搜索、用户动态、评论)"diff --git a/agent_reach/channels/youtube.py b/agent_reach/channels/youtube.pyindex a1858cf8..5ee1e2b8 100644--- a/agent_reach/channels/youtube.py+++ b/agent_reach/channels/youtube.py@@ -9,6 +9,16 @@ from agent_reach.utils.text import read_utf8_textfrom .base import Channel+def _has_js_runtime_config(config_path) -> bool:+ """Return whether yt-dlp config explicitly enables a JS runtime."""+ try:+ if not config_path.exists():+ return False+ return "--js-runtimes" in read_utf8_text(config_path)+ except OSError:+ return False++class YouTubeChannel(Channel):name = "youtube"description = "YouTube 视频和字幕"@@ -36,10 +46,7 @@ class YouTubeChannel(Channel):has_deno = shutil.which("deno")if not has_deno:ytdlp_config = get_ytdlp_config_path()- has_js_config = False- if ytdlp_config.exists():- has_js_config = "--js-runtimes" in read_utf8_text(ytdlp_config)- if not has_js_config:+ if not _has_js_runtime_config(ytdlp_config):return "warn", (f"yt-dlp 已安装但未配置 JS runtime。运行:\n {render_ytdlp_fix_command()}")diff --git a/agent_reach/cli.py b/agent_reach/cli.pyindex 509493b1..173f98d6 100644--- a/agent_reach/cli.py+++ b/agent_reach/cli.py@@ -763,6 +763,8 @@ def _install_weibo_deps():import shutilimport subprocess+ from agent_reach.utils.process import mcporter_utf8_env_args, utf8_subprocess_env+print("Setting up Weibo MCP server...")# Check if already installed and working@@ -771,7 +773,8 @@ def _install_weibo_deps():try:r = subprocess.run([mcporter, "config", "list"], capture_output=True,- encoding="utf-8", errors="replace", timeout=5+ encoding="utf-8", errors="replace", timeout=5,+ env=utf8_subprocess_env(),)if "weibo" in r.stdout:print(" ✅ Weibo MCP already configured")@@ -784,25 +787,27 @@ def _install_weibo_deps():subprocess.run([sys.executable, "-m", "pip", "install", "-q","git+https://github.com/Panniantong/mcp-server-weibo.git"],- check=True, timeout=120+ check=True, timeout=120, env=utf8_subprocess_env())print(" ✅ mcp-server-weibo installed (Panniantong fork)")except Exception as e:print(f" [!] mcp-server-weibo install failed: {e}")return- # Register with mcporter+ # Register with mcporter (force UTF-8 in the server's env — Windows GBK+ # consoles otherwise corrupt the MCP server's Chinese output)if mcporter:try:subprocess.run(- [mcporter, "config", "add", "weibo", "--command", "mcp-server-weibo"],+ [mcporter, "config", "add", "weibo", "--command", "mcp-server-weibo",+ *mcporter_utf8_env_args()],check=True, capture_output=True, timeout=10)print(" ✅ Weibo MCP registered with mcporter")except Exception:- print(" [!] mcporter config add failed. Run manually: mcporter config add weibo --command 'mcp-server-weibo'")+ print(" [!] mcporter config add failed. Run manually: mcporter config add weibo --command 'mcp-server-weibo' --env PYTHONUTF8=1 --env PYTHONIOENCODING=utf-8")else:- print(" -- mcporter not found, skipping MCP registration. Install mcporter first, then run: mcporter config add weibo --command 'mcp-server-weibo'")+ print(" -- mcporter not found, skipping MCP registration. Install mcporter first, then run: mcporter config add weibo --command 'mcp-server-weibo' --env PYTHONUTF8=1 --env PYTHONIOENCODING=utf-8")def _install_wechat_deps():diff --git a/agent_reach/utils/process.py b/agent_reach/utils/process.pynew file mode 100644index 00000000..27766163--- /dev/null+++ b/agent_reach/utils/process.py@@ -0,0 +1,26 @@+"""Subprocess helpers for consistent cross-platform text handling."""++from __future__ import annotations++import os+from collections.abc import Mapping++UTF8_ENV = {+ "PYTHONUTF8": "1",+ "PYTHONIOENCODING": "utf-8",+}+++def utf8_subprocess_env(base: Mapping[str, str] | None = None) -> dict[str, str]:+ """Return an environment that forces Python child processes into UTF-8 mode."""+ env = dict(base or os.environ)+ env.update(UTF8_ENV)+ return env+++def mcporter_utf8_env_args() -> list[str]:+ """Return mcporter --env arguments for UTF-8 Python stdio servers."""+ args = []+ for key, value in UTF8_ENV.items():+ args.extend(["--env", f"{key}={value}"])+ return argsdiff --git a/tests/test_process.py b/tests/test_process.pynew file mode 100644index 00000000..550acfb9--- /dev/null+++ b/tests/test_process.py@@ -0,0 +1,18 @@+from agent_reach.utils.process import mcporter_utf8_env_args, utf8_subprocess_env+++def test_utf8_subprocess_env_forces_python_utf8():+ env = utf8_subprocess_env({"PYTHONUTF8": "0", "OTHER": "value"})++ assert env["PYTHONUTF8"] == "1"+ assert env["PYTHONIOENCODING"] == "utf-8"+ assert env["OTHER"] == "value"+++def test_mcporter_utf8_env_args():+ assert mcporter_utf8_env_args() == [+ "--env",+ "PYTHONUTF8=1",+ "--env",+ "PYTHONIOENCODING=utf-8",+ ]diff --git a/tests/test_skill_command.py b/tests/test_skill_command.pyindex 38c1943e..7ef13523 100644--- a/tests/test_skill_command.py+++ b/tests/test_skill_command.py@@ -39,14 +39,11 @@ class TestSkillCommand(unittest.TestCase):with patch.dict(os.environ, env, clear=True):_install_skill()- target = os.path.join(skill_dir, "agent-reach", "SKILL.md")# Check at least one known skill dir pattern- found = Falsefor dirpath, _, filenames in os.walk(tmpdir):if "SKILL.md" in filenames:- found = True# Verify content is non-empty- with open(os.path.join(dirpath, "SKILL.md")) as f:+ with open(os.path.join(dirpath, "SKILL.md"), encoding="utf-8") as f:content = f.read()self.assertIn("Agent Reach", content)# _install_skill may or may not find dirs depending on mock; just ensure no crash@@ -58,7 +55,7 @@ class TestSkillCommand(unittest.TestCase):# Create a fake skill installationskill_path = os.path.join(tmpdir, ".openclaw", "skills", "agent-reach")os.makedirs(skill_path)- with open(os.path.join(skill_path, "SKILL.md"), "w") as f:+ with open(os.path.join(skill_path, "SKILL.md"), "w", encoding="utf-8") as f:f.write("test")self.assertTrue(os.path.exists(skill_path))@@ -92,7 +89,7 @@ class TestSkillCommand(unittest.TestCase):target = os.path.join(skill_parent, "agent-reach", "SKILL.md")self.assertTrue(os.path.exists(target))- with open(target) as f:+ with open(target, encoding="utf-8") as f:content = f.read()self.assertIn("Agent Reach", content)@@ -114,7 +111,7 @@ class TestSkillCommand(unittest.TestCase):target = os.path.join(skill_parent, "agent-reach", "SKILL.md")self.assertTrue(os.path.exists(target))- with open(target) as f:+ with open(target, encoding="utf-8") as f:content = f.read()self.assertTrue(content.strip())self.assertIn("WeChat Articles, Xiaoyuzhou Podcast", content)