Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.
Python agent-infrastructureai-agentai-searchautomationbilibiliclaude-codeclicursorfree-apillm-toolsmcppythonreddit-scrapertwitter-scraperweb-scraperxiaohongshuyoutube-transcript
fix(security): enforce least-privilege credential boundaries
Pnantfe58c3fe46ce
diff --git a/agent_reach/channels/_opencli_site.py b/agent_reach/channels/_opencli_site.pyindex 8e93766f..52b2aff2 100644--- a/agent_reach/channels/_opencli_site.py+++ b/agent_reach/channels/_opencli_site.py@@ -1,7 +1,7 @@# -*- coding: utf-8 -*-"""Shared channel helper for OpenCLI browser-session-only platforms."""-from urllib.parse import urlparse+from agent_reach.utils.url import host_matchesfrom .base import Channel@@ -22,8 +22,7 @@ class OpenCLISiteChannel(Channel):tier = 1def can_handle(self, url: str) -> bool:- domain = urlparse(url).netloc.lower()- return any(domain == d or domain.endswith(f".{d}") for d in self.domains)+ return host_matches(url, *self.domains)def check(self, config=None):from agent_reach.backends import opencli_statusdiff --git a/agent_reach/channels/bilibili.py b/agent_reach/channels/bilibili.pyindex d7aed8f1..6d3c0f6c 100644--- a/agent_reach/channels/bilibili.py+++ b/agent_reach/channels/bilibili.py@@ -39,9 +39,9 @@ class BilibiliChannel(Channel):tier = 1def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- d = urlparse(url).netloc.lower()- return "bilibili.com" in d or "b23.tv" in d+ from agent_reach.utils.url import host_matches++ return host_matches(url, "bilibili.com", "b23.tv")def check(self, config=None):"""Probe candidates in order; first fully-usable backend wins."""diff --git a/agent_reach/channels/github.py b/agent_reach/channels/github.pyindex 1908a98b..62f695f5 100644--- a/agent_reach/channels/github.py+++ b/agent_reach/channels/github.py@@ -13,8 +13,9 @@ class GitHubChannel(Channel):tier = 0def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- return "github.com" in urlparse(url).netloc.lower()+ from agent_reach.utils.url import host_matches++ return host_matches(url, "github.com")def check(self, config=None):# 真跑 gh auth status 探活。注意:未登录时 rc!=0 是正常业务态(warn),不是 error。diff --git a/agent_reach/channels/linkedin.py b/agent_reach/channels/linkedin.pyindex a2a5ddae..d480ae93 100644--- a/agent_reach/channels/linkedin.py+++ b/agent_reach/channels/linkedin.py@@ -18,8 +18,9 @@ class LinkedInChannel(Channel):tier = 2def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- return "linkedin.com" in urlparse(url).netloc.lower()+ from agent_reach.utils.url import host_matches++ return host_matches(url, "linkedin.com")def check(self, config=None):self.active_backend = Nonediff --git a/agent_reach/channels/reddit.py b/agent_reach/channels/reddit.pyindex cece520a..7754074f 100644--- a/agent_reach/channels/reddit.py+++ b/agent_reach/channels/reddit.py@@ -39,10 +39,9 @@ class RedditChannel(Channel):tier = 1 # no zero-config path exists — see module docstringdef can_handle(self, url: str) -> bool:- from urllib.parse import urlparse+ from agent_reach.utils.url import host_matches- d = urlparse(url).netloc.lower()- return "reddit.com" in d or "redd.it" in d+ return host_matches(url, "reddit.com", "redd.it")def check(self, config=None):"""Probe candidates in order; first fully-usable backend wins."""diff --git a/agent_reach/channels/twitter.py b/agent_reach/channels/twitter.pyindex 0c444db7..ebb02d5a 100644--- a/agent_reach/channels/twitter.py+++ b/agent_reach/channels/twitter.py@@ -4,6 +4,7 @@import osfrom agent_reach.probe import probe_command+from agent_reach.utils.url import host_matchesfrom .base import Channel@@ -37,9 +38,7 @@ class TwitterChannel(Channel):tier = 1def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- d = urlparse(url).netloc.lower()- return "x.com" in d or "twitter.com" in d+ return host_matches(url, "x.com", "twitter.com")def check(self, config=None):"""Probe candidates in order; first fully-usable backend wins.diff --git a/agent_reach/channels/v2ex.py b/agent_reach/channels/v2ex.pyindex 1b24561a..b6032522 100644--- a/agent_reach/channels/v2ex.py+++ b/agent_reach/channels/v2ex.py@@ -4,6 +4,9 @@import jsonimport urllib.requestfrom typing import Any++from agent_reach.utils.text import scrub_url_credentials+from .base import Channel_UA = "agent-reach/1.0"@@ -28,9 +31,9 @@ class V2EXChannel(Channel):# ------------------------------------------------------------------ #def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- d = urlparse(url).netloc.lower()- return "v2ex.com" in d+ from agent_reach.utils.url import host_matches++ return host_matches(url, "v2ex.com")# ------------------------------------------------------------------ ## Health check@@ -45,7 +48,10 @@ class V2EXChannel(Channel):return "ok", "公开 API 可用(热门主题、节点浏览、主题详情、用户信息)"except Exception as e:self.active_backend = None- return "warn", f"V2EX API 连接失败(可能需要代理):{e}"+ return (+ "warn",+ f"V2EX API 连接失败(可能需要代理):{scrub_url_credentials(e)}",+ )# ------------------------------------------------------------------ ## Data-fetching methodsdiff --git a/agent_reach/channels/xiaohongshu.py b/agent_reach/channels/xiaohongshu.pyindex f60ab807..ab651e74 100644--- a/agent_reach/channels/xiaohongshu.py+++ b/agent_reach/channels/xiaohongshu.py@@ -153,9 +153,9 @@ class XiaoHongShuChannel(Channel):tier = 1def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- d = urlparse(url).netloc.lower()- return "xiaohongshu.com" in d or "xhslink.com" in d+ from agent_reach.utils.url import host_matches++ return host_matches(url, "xiaohongshu.com", "xhslink.com")def check(self, config=None):"""Probe candidates in order; first fully-usable backend wins.diff --git a/agent_reach/channels/xiaoyuzhou.py b/agent_reach/channels/xiaoyuzhou.pyindex efc73b78..5cd50d5c 100644--- a/agent_reach/channels/xiaoyuzhou.py+++ b/agent_reach/channels/xiaoyuzhou.py@@ -2,8 +2,10 @@"""Xiaoyuzhou Podcast (小宇宙播客) — transcribe podcasts via Groq Whisper API."""import os+from agent_reach.config import Configfrom agent_reach.probe import probe_command+from .base import Channel@@ -14,9 +16,9 @@ class XiaoyuzhouChannel(Channel):tier = 1def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse- d = urlparse(url).netloc.lower()- return "xiaoyuzhoufm.com" in d+ from agent_reach.utils.url import host_matches++ return host_matches(url, "xiaoyuzhoufm.com")def check(self, config=None):self.active_backend = Nonediff --git a/agent_reach/channels/xueqiu.py b/agent_reach/channels/xueqiu.pyindex a87a73b4..1e65d866 100644--- a/agent_reach/channels/xueqiu.py+++ b/agent_reach/channels/xueqiu.py@@ -122,8 +122,9 @@ class XueqiuChannel(Channel):# ------------------------------------------------------------------ #def can_handle(self, url: str) -> bool:- d = urllib.parse.urlparse(url).netloc.lower()- return "xueqiu.com" in d+ from agent_reach.utils.url import host_matches++ return host_matches(url, "xueqiu.com")# ------------------------------------------------------------------ ## Health check@@ -142,10 +143,13 @@ class XueqiuChannel(Channel):return "ok", "公开 API 可用(行情、搜索、热帖、热股)"return "warn", "API 响应异常(返回数据为空)"except Exception as e:- detail = str(e).rstrip(": ")+ from agent_reach.utils.text import scrub_url_credentials++ detail = scrub_url_credentials(e).rstrip(": ")return "warn", (f"Xueqiu API 连接失败:{detail}。"- "如需登录 Cookie,请通过显式配置流程保存后重试;"+ "如需登录 Cookie,请运行:agent-reach configure "+ "--from-browser chrome --platform xueqiu;""doctor 不会自动读取浏览器 Cookie。")diff --git a/agent_reach/channels/youtube.py b/agent_reach/channels/youtube.pyindex 90ec5146..954a58ea 100644--- a/agent_reach/channels/youtube.py+++ b/agent_reach/channels/youtube.py@@ -37,10 +37,9 @@ class YouTubeChannel(Channel):tier = 0def can_handle(self, url: str) -> bool:- from urllib.parse import urlparse+ from agent_reach.utils.url import host_matches- d = urlparse(url).netloc.lower()- return "youtube.com" in d or "youtu.be" in d+ return host_matches(url, "youtube.com", "youtu.be")def check(self, config=None):# 真跑 yt-dlp --version 探活,区分未装 / venv 断链 / 跑不动diff --git a/agent_reach/cookie_extract.py b/agent_reach/cookie_extract.pyindex a3e92876..03a90085 100644--- a/agent_reach/cookie_extract.py+++ b/agent_reach/cookie_extract.py@@ -1,77 +1,263 @@# -*- coding: utf-8 -*--"""Auto-extract cookies from local browsers for all supported platforms.+"""Least-privilege cookie extraction from local browsers.Supports: Chrome, Firefox, Edge, Brave, Opera-Extracts: Twitter, XiaoHongShu, Bilibili cookies in one shot.+Extracts one explicitly requested platform at a time.Usage:- agent-reach configure --from-browser chrome+ agent-reach configure --from-browser chrome --platform xueqiu"""-from typing import Dict, List, Tuple+from dataclasses import dataclass+from pathlib import Path+from typing import Dict, Iterator, List, Optional, Tuple, TypedDict-# Platform cookie specs: (platform_name, domain_pattern, needed_cookies)-PLATFORM_SPECS = [+from agent_reach.utils.text import scrub_url_credentials+from agent_reach.utils.url import domain_matches+++class PlatformSpec(TypedDict):+ name: str+ domains: Tuple[str, ...]+ cookies: Optional[Tuple[str, ...]]+ config_key: str+++class ChromiumPaths(TypedDict):+ darwin: str+ linux: str+ win32: Tuple[str, ...]+++PLATFORM_SPECS: Tuple[PlatformSpec, ...] = ({"name": "Twitter/X",- "domains": [".x.com", ".twitter.com"],- "cookies": ["auth_token", "ct0"],+ "domains": (".x.com", ".twitter.com"),+ "cookies": ("auth_token", "ct0"),"config_key": "twitter",},{"name": "XiaoHongShu",- "domains": [".xiaohongshu.com"],- "cookies": None, # None = grab all cookies as header string+ "domains": (".xiaohongshu.com",),+ "cookies": None, # manual Cookie-Editor export only"config_key": "xhs",},{"name": "Bilibili",- "domains": [".bilibili.com"],- "cookies": ["SESSDATA", "bili_jct"],+ "domains": (".bilibili.com",),+ "cookies": ("SESSDATA", "bili_jct"),"config_key": "bilibili",},{"name": "Xueqiu",- "domains": [".xueqiu.com", "xueqiu.com"],- "cookies": None, # grab all — xq_a_token + session cookies required+ "domains": (".xueqiu.com",),+ "cookies": ("xq_a_token",),"config_key": "xueqiu",},-]+)++_PLATFORM_SPECS_BY_KEY: Dict[str, PlatformSpec] = {+ spec["config_key"]: spec for spec in PLATFORM_SPECS+}+SUPPORTED_BROWSERS = ("chrome", "firefox", "edge", "brave", "opera")+PROFILE_SELECTABLE_BROWSERS = ("chrome", "edge", "brave")+_COOKIE_EDITOR_ONLY = {+ "twitter": "twitter-cookies",+ "xhs": "xhs-cookies",+}++_CHROMIUM_USER_DATA_DIRS: Dict[str, ChromiumPaths] = {+ "chrome": {+ "darwin": "~/Library/Application Support/Google/Chrome",+ "linux": "~/.config/google-chrome",+ "win32": ("Google", "Chrome", "User Data"),+ },+ "edge": {+ "darwin": "~/Library/Application Support/Microsoft Edge",+ "linux": "~/.config/microsoft-edge",+ "win32": ("Microsoft", "Edge", "User Data"),+ },+ "brave": {+ "darwin": "~/Library/Application Support/BraveSoftware/Brave-Browser",+ "linux": "~/.config/BraveSoftware/Brave-Browser",+ "win32": ("BraveSoftware", "Brave-Browser", "User Data"),+ },+}+++@dataclass(frozen=True)+class BrowserConfigResult:+ """One browser configuration outcome with non-secret write targets."""++ platform: str+ success: bool+ message: str+ targets: Tuple[str, ...] = ()++ def __iter__(self) -> Iterator[object]:+ """Preserve the historical three-value unpacking API."""+ yield self.platform+ yield self.success+ yield self.message+++def _chromium_user_data_dir(browser: str) -> Optional[Path]:+ """Return the browser's profile root on the current operating system."""+ import os+ import sys++ paths = _CHROMIUM_USER_DATA_DIRS.get(browser)+ if paths is None:+ return None+ if sys.platform == "win32":+ local_appdata = os.environ.get("LOCALAPPDATA")+ if not local_appdata:+ return None+ return Path(local_appdata).joinpath(*paths["win32"])+ path = paths["darwin"] if sys.platform == "darwin" else paths["linux"]+ return Path(os.path.expanduser(path))+++def list_browser_profiles(browser: str = "chrome") -> List[Dict[str, str]]:+ """List named Chromium profiles that contain a cookie database."""+ browser = browser.lower()+ root = _chromium_user_data_dir(browser)+ if root is None:+ return []+ root = Path(root)+ if not root.is_dir():+ return []++ profiles = []+ for profile_dir in root.iterdir():+ if not profile_dir.is_dir():+ continue+ cookie_file = profile_dir / "Network" / "Cookies"+ if not cookie_file.is_file():+ cookie_file = profile_dir / "Cookies"+ if cookie_file.is_file():+ profiles.append(+ {+ "folder": profile_dir.name,+ "cookies_path": str(cookie_file),+ }+ )++ def sort_key(item):+ folder = item["folder"]+ suffix = folder.rsplit(" ", 1)[-1]+ number = int(suffix) if suffix.isdigit() else 0+ return (folder != "Default", number, folder)+ profiles.sort(key=sort_key)+ return profiles-def extract_all(browser: str = "chrome") -> Dict[str, dict]:++def _profile_cookie_file(browser: str, profile: str) -> str:+ """Resolve an explicit profile or fail without falling back to Default."""+ if browser not in PROFILE_SELECTABLE_BROWSERS:+ raise ValueError(+ "Profile selection is supported only for "+ f"{', '.join(PROFILE_SELECTABLE_BROWSERS)}, "+ f"not {scrub_url_credentials(browser)}."+ )++ profiles = list_browser_profiles(browser)+ for candidate in profiles:+ if candidate["folder"] == profile:+ return candidate["cookies_path"]++ available = ", ".join(+ scrub_url_credentials(item["folder"]) for item in profiles+ )+ hint = f" Available profiles: {available}." if available else ""+ raise ValueError(+ f"Profile '{scrub_url_credentials(profile)}' not found for "+ f"{scrub_url_credentials(browser)}.{hint}"+ )+++def _platform_spec(platform: Optional[str]) -> PlatformSpec:+ """Return the one explicitly requested platform specification."""+ if not platform:+ raise ValueError(+ "platform is required for browser-cookie extraction; "+ f"choose one of: {', '.join(_PLATFORM_SPECS_BY_KEY)}"+ )+ key = platform.lower()+ try:+ return _PLATFORM_SPECS_BY_KEY[key]+ except KeyError as exc:+ raise ValueError(+ f"Unsupported platform: {scrub_url_credentials(platform)}. "+ f"Supported: {', '.join(_PLATFORM_SPECS_BY_KEY)}"+ ) from exc+++def _require_browser_extractable(spec: PlatformSpec) -> None:+ """Reject platforms whose project policy requires a manual cookie export."""+ manual_key = _COOKIE_EDITOR_ONLY.get(spec["config_key"])+ if manual_key:+ raise ValueError(+ f"Automatic browser extraction is disabled for {spec['name']}. "+ "Export the required cookies with Cookie-Editor, then use "+ f"`agent-reach configure {manual_key}`."+ )+++def extract_all(+ browser: str = "chrome",+ *,+ platform: Optional[str] = None,+ profile: Optional[str] = None,+) -> Dict[str, dict]:"""- Extract cookies for all supported platforms from the specified browser.-+ Extract cookies for one explicitly requested platform.++ The legacy function name is retained for API compatibility, but an+ all-platform read is intentionally no longer supported.+Returns:- {- "twitter": {"auth_token": "xxx", "ct0": "yyy"},- "xhs": {"cookie_string": "a=1; b=2; ..."},- "bilibili": {"SESSDATA": "xxx", "bili_jct": "yyy"},- }+ {"xueqiu": {"xq_a_token": "xxx"}}"""+ spec = _platform_spec(platform)+ _require_browser_extractable(spec)+ browser = browser.lower()+ if browser not in SUPPORTED_BROWSERS:+ raise ValueError(+ f"Unsupported browser: {scrub_url_credentials(browser)}. "+ f"Supported: {', '.join(SUPPORTED_BROWSERS)}"+ )+ cookie_file = _profile_cookie_file(browser, profile) if profile else None+ needed_cookies = spec["cookies"]+ if needed_cookies is None:+ raise ValueError(+ f"Automatic full-domain extraction is disabled for {spec['name']}."+ )+# Try rookiepy first (Rust-based, more stable), fallback to browser_cookie3use_rookiepy = False- try:- import rookiepy- use_rookiepy = True- except ImportError:+ if cookie_file is None:+ try:+ import rookiepy+ use_rookiepy = True+ except ImportError:+ pass+ if not use_rookiepy:try:import browser_cookie3except ImportError:+ profile_hint = (+ f" for profile '{scrub_url_credentials(profile)}'"+ if profile is not None+ else ""+ )raise RuntimeError(- "Cookie extraction requires rookiepy or browser_cookie3.\n"- "Install: pip install rookiepy (recommended)\n"- " or: pip install browser-cookie3"+ f"Cookie extraction{profile_hint} requires browser_cookie3"+ " (or rookiepy when no profile is selected).\n"+ "Install: pip install browser-cookie3")- browser = browser.lower()- supported = ["chrome", "firefox", "edge", "brave", "opera"]- if browser not in supported:- raise ValueError(- f"Unsupported browser: {browser}. Supported: {', '.join(supported)}"- )-if use_rookiepy:# rookiepy returns list of dicts with name/value/domain/path keystry:@@ -82,7 +268,7 @@ def extract_all(browser: str = "chrome") -> Dict[str, dict]:"brave": rookiepy.brave,"opera": rookiepy.opera,}- raw_cookies = browser_funcs[browser]()+ raw_cookies = browser_funcs[browser](list(spec["domains"]))# Wrap into objects with .name, .value, .domain for compatibilityclass _Cookie:def __init__(self, d):@@ -92,7 +278,8 @@ def extract_all(browser: str = "chrome") -> Dict[str, dict]:cookie_jar = [_Cookie(c) for c in raw_cookies]except Exception as e:raise RuntimeError(- f"Could not read {browser} cookies via rookiepy: {e}\n"+ f"Could not read {browser} cookies via rookiepy: "+ f"{scrub_url_credentials(e)}\n"f"Make sure {browser} is closed and you have permission.")else:@@ -104,44 +291,41 @@ def extract_all(browser: str = "chrome") -> Dict[str, dict]:"opera": browser_cookie3.opera,}try:- cookie_jar = browser_funcs[browser]()+ cookie_jar = []+ seen = set()+ for domain in spec["domains"]:+ kwargs = {"domain_name": domain}+ if cookie_file is not None:+ kwargs["cookie_file"] = cookie_file+ for cookie in browser_funcs[browser](**kwargs):+ identity = (+ getattr(cookie, "name", ""),+ getattr(cookie, "domain", ""),+ getattr(cookie, "path", ""),+ getattr(cookie, "value", ""),+ )+ if identity not in seen:+ seen.add(identity)+ cookie_jar.append(cookie)except Exception as e:raise RuntimeError(- f"Could not read {browser} cookies: {e}\n"+ f"Could not read {browser} cookies: {scrub_url_credentials(e)}\n"f"Make sure {browser} is closed and you have permission.")results = {}- for spec in PLATFORM_SPECS:- platform_cookies = {}- all_cookies_for_domain = []-- for cookie in cookie_jar:- # Check if cookie belongs to this platform- domain_match = any(- cookie.domain.endswith(d) or cookie.domain == d.lstrip(".")- for d in spec["domains"]- )- if not domain_match:- continue-- all_cookies_for_domain.append(cookie)+ platform_cookies = {}+ for cookie in cookie_jar:+ # Re-check returned cookies instead of trusting the backend filter.+ if not domain_matches(cookie.domain, *spec["domains"]):+ continue- if spec["cookies"] is not None:- if cookie.name in spec["cookies"]:- platform_cookies[cookie.name] = cookie.value+ if cookie.name in needed_cookies:+ platform_cookies[cookie.name] = cookie.value- if spec["cookies"] is None:- # Grab all as header string- if all_cookies_for_domain:- cookie_str = "; ".join(- f"{c.name}={c.value}" for c in all_cookies_for_domain- )- results[spec["config_key"]] = {"cookie_string": cookie_str}- else:- if platform_cookies:- results[spec["config_key"]] = platform_cookies+ if platform_cookies:+ results[spec["config_key"]] = platform_cookiesreturn results@@ -173,7 +357,7 @@ def _open_owner_only(path: str):return handle-def _sync_xfetch_session(auth_token: str, ct0: str) -> None:+def _sync_xfetch_session(auth_token: str, ct0: str) -> bool:"""Sync Twitter credentials to ~/.config/xfetch/session.json (legacy xreach compat)."""import jsonimport os@@ -195,12 +379,13 @@ def _sync_xfetch_session(auth_token: str, ct0: str) -> None:session_data["ct0"] = ct0with _open_owner_only(session_path) as sf:json.dump(session_data, sf, indent=2)+ return Trueexcept Exception:# Non-fatal: agent-reach config is the source of truth, xfetch sync is best-effort- pass+ return False-def _sync_bird_env(auth_token: str, ct0: str) -> None:+def _sync_bird_env(auth_token: str, ct0: str) -> bool:"""Write Twitter credentials to ~/.config/bird/credentials.env for bird CLI.bird reads AUTH_TOKEN and CT0 from environment variables. This writes a@@ -220,78 +405,106 @@ def _sync_bird_env(auth_token: str, ct0: str) -> None:with _open_owner_only(env_path) as f:f.write(f"AUTH_TOKEN={shlex.quote(auth_token)}\n")f.write(f"CT0={shlex.quote(ct0)}\n")+ return Trueexcept Exception:# Non-fatal: agent-reach config is the source of truth, bird env sync is best-effort- pass+ return False# Alias for callers expecting the name _sync_bird_credentials_sync_bird_credentials = _sync_bird_env-def configure_from_browser(browser: str, config) -> List[Tuple[str, bool, str]]:+def configure_from_browser(+ browser: str,+ config,+ *,+ platform: Optional[str] = None,+ profile: Optional[str] = None,+) -> List[BrowserConfigResult]:"""- Extract cookies and configure all found platforms.-- Returns list of (platform_name, success, message) tuples.+ Extract and configure exactly one explicitly selected platform.++ Result objects still unpack as ``(platform, success, message)`` and expose+ a ``targets`` attribute so the CLI can disclose every non-secret config key+ or legacy path written."""- results_list = []+ spec = _platform_spec(platform)+ _require_browser_extractable(spec)+ results_list: List[BrowserConfigResult] = []try:- extracted = extract_all(browser)+ extracted = extract_all(+ browser,+ platform=spec["config_key"],+ profile=profile,+ )+ except ValueError:+ raiseexcept Exception as e:- return [("Browser", False, str(e))]-- if not extracted:- return [("All platforms", False,- f"No platform cookies found in {browser}. "- f"Make sure you're logged into Twitter, XiaoHongShu, etc. in {browser}.")]-- # Configure each found platform- if "twitter" in extracted:- tc = extracted["twitter"]- if "auth_token" in tc and "ct0" in tc:- config.set("twitter_auth_token", tc["auth_token"])- config.set("twitter_ct0", tc["ct0"])- # Legacy sync (best-effort)- _sync_xfetch_session(tc["auth_token"], tc["ct0"])- results_list.append(("Twitter/X", True, "auth_token + ct0"))- else:- found = ", ".join(tc.keys())- missing = [k for k in ["auth_token", "ct0"] if k not in tc]- results_list.append(("Twitter/X", False,- f"Found {found}, but missing: {', '.join(missing)}. "- f"Make sure you're logged into x.com in {browser}."))-- if "xhs" in extracted:- cookie_str = extracted["xhs"].get("cookie_string", "")- if cookie_str:- config.set("xhs_cookie", cookie_str)- n_cookies = len(cookie_str.split(";"))- results_list.append(("XiaoHongShu", True, f"{n_cookies} cookies"))-- if "bilibili" in extracted:+ return [+ BrowserConfigResult(+ "Browser", False, scrub_url_credentials(e)+ )+ ]++ config_key = spec["config_key"]+ if config_key not in extracted:+ return [+ BrowserConfigResult(+ spec["name"],+ False,+ f"No {spec['name']} cookies found in {browser}. "+ f"Make sure you're logged into the selected platform.",+ )+ ]++ if config_key == "bilibili":bc = extracted["bilibili"]if "SESSDATA" in bc:config.set("bilibili_sessdata", bc["SESSDATA"])+ targets = ["bilibili_sessdata"]if "bili_jct" in bc:config.set("bilibili_csrf", bc["bili_jct"])- results_list.append(("Bilibili", True, "SESSDATA" +- (" + bili_jct" if "bili_jct" in bc else "")))+ targets.append("bilibili_csrf")+ results_list.append(+ BrowserConfigResult(+ "Bilibili",+ True,+ "SESSDATA" + (" + bili_jct" if "bili_jct" in bc else ""),+ tuple(targets),+ )+ )else:- results_list.append(("Bilibili", False,- f"No SESSDATA found. Make sure you're logged into bilibili.com in {browser}."))+ results_list.append(+ BrowserConfigResult(+ "Bilibili",+ False,+ f"No SESSDATA found. "+ f"Make sure you're logged into bilibili.com in {browser}.",+ )+ )- if "xueqiu" in extracted:- cookie_str = extracted["xueqiu"].get("cookie_string", "")- # Only save if xq_a_token is present — anonymous cookies are useless- if cookie_str and "xq_a_token" in cookie_str:+ elif config_key == "xueqiu":+ token = extracted["xueqiu"].get("xq_a_token", "")+ if token:+ cookie_str = f"xq_a_token={token}"config.set("xueqiu_cookie", cookie_str)- n_cookies = len(cookie_str.split(";"))- results_list.append(("Xueqiu", True, f"{n_cookies} cookies (含 xq_a_token)"))- elif cookie_str:- results_list.append(("Xueqiu", False,- f"找到 {len(cookie_str.split(';'))} 个 Cookie 但缺少 xq_a_token,"- f"请先在 {browser} 中登录 xueqiu.com"))+ results_list.append(+ BrowserConfigResult(+ "Xueqiu",+ True,+ "xq_a_token",+ ("xueqiu_cookie",),+ )+ )+ else:+ results_list.append(+ BrowserConfigResult(+ "Xueqiu",+ False,+ f"未找到 xq_a_token,请先在 {browser} 中登录 xueqiu.com",+ )+ )return results_listdiff --git a/agent_reach/utils/text.py b/agent_reach/utils/text.pyindex 850a96d9..9ab61b99 100644--- a/agent_reach/utils/text.py+++ b/agent_reach/utils/text.py@@ -2,8 +2,31 @@from __future__ import annotations+import refrom pathlib import Path+_URL_CREDENTIALS_RE = re.compile(+ r"([A-Za-z][A-Za-z0-9+.\-]{0,19}://)[^/\s@]+@"+)+_BARE_USERINFO_RE = re.compile(+ r"(?<![A-Za-z0-9._%+\-])[^:/\s@]+:[^/\s@]+@(?=[A-Za-z0-9.\-\[])"+)+_URL_QUERY_SECRET_RE = re.compile(+ r"([?&#](?:"+ r"access[_-]?token|auth[_-]?token|token|bearer|"+ r"api[_-]?key|key|password|passwd|secret|"+ r"signature|sig|session(?:id)?|cookie|credential"+ r")=)[^&#\s]*",+ re.IGNORECASE,+)+++def scrub_url_credentials(text: object) -> str:+ """Redact URL userinfo and sensitive query/fragment values from text."""+ scrubbed = _URL_CREDENTIALS_RE.sub(r"\1***@", str(text))+ scrubbed = _BARE_USERINFO_RE.sub("***@", scrubbed)+ return _URL_QUERY_SECRET_RE.sub(r"\1***", scrubbed)+def read_utf8_text(path: str | Path, default: str = "") -> str:"""Read text as UTF-8 with replacement semantics."""diff --git a/agent_reach/utils/url.py b/agent_reach/utils/url.pynew file mode 100644index 00000000..e1cfd354--- /dev/null+++ b/agent_reach/utils/url.py@@ -0,0 +1,38 @@+"""Security helpers for untrusted URLs."""++from __future__ import annotations++from urllib.parse import urlsplit+++def domain_matches(host: str, *domains: str) -> bool:+ """Match a hostname/cookie domain exactly or as a real subdomain."""+ normalized_host = str(host or "").lower().lstrip(".").rstrip(".")+ if not normalized_host:+ return False+ for domain in domains:+ allowed = domain.lower().lstrip(".").rstrip(".")+ if normalized_host == allowed or normalized_host.endswith("." + allowed):+ return True+ return False+++def host_matches(url: str, *domains: str) -> bool:+ """Return whether *url* has an exact allowed host or a real subdomain.++ Only HTTP(S) URLs without userinfo are accepted. Using ``hostname`` rather+ than substring matching prevents lookalikes such as ``x.com.evil.test`` and+ userinfo disguises such as ``[email protected]``.+ """+ try:+ parsed = urlsplit(url)+ host = (parsed.hostname or "").lower().rstrip(".")+ except (TypeError, ValueError):+ return False++ if parsed.scheme.lower() not in {"http", "https"}:+ return False+ if not host or parsed.username is not None or parsed.password is not None:+ return False++ return domain_matches(host, *domains)diff --git a/tests/test_channels.py b/tests/test_channels.pyindex 82b58864..ff5c3e94 100644--- a/tests/test_channels.py+++ b/tests/test_channels.py@@ -701,6 +701,7 @@ class TestXueqiuChannel:"open",lambda req, timeout=None: requested.append(req.full_url) or FakeResponse(),)+ assert {cookie.name for cookie in xueqiu_mod._cookie_jar} == {"xq_a_token"}xueqiu_mod._ensure_cookies()diff --git a/tests/test_cookie_security.py b/tests/test_cookie_security.pynew file mode 100644index 00000000..0e1ab4c4--- /dev/null+++ b/tests/test_cookie_security.py@@ -0,0 +1,222 @@+"""Least-privilege browser-cookie extraction and URL routing regressions."""++from __future__ import annotations++import sys+from types import SimpleNamespace++import pytest++from agent_reach import cookie_extract+++class RecordingConfig:+ def __init__(self):+ self.values = {}++ def set(self, key, value):+ self.values[key] = value+++def test_browser_extraction_requires_an_explicit_platform():+ with pytest.raises(ValueError, match="platform"):+ cookie_extract.extract_all("chrome")+++def test_rookiepy_is_limited_at_source_to_the_requested_platform(monkeypatch):+ calls = []++ def chrome(domains):+ calls.append(domains)+ return [+ {"name": "SESSDATA", "value": "session", "domain": ".bilibili.com"},+ {"name": "bili_jct", "value": "csrf", "domain": ".bilibili.com"},+ ]++ fake_rookiepy = SimpleNamespace(+ chrome=chrome,+ firefox=lambda domains: [],+ edge=lambda domains: [],+ brave=lambda domains: [],+ opera=lambda domains: [],+ )+ monkeypatch.setitem(sys.modules, "rookiepy", fake_rookiepy)++ extracted = cookie_extract.extract_all("chrome", platform="bilibili")++ assert calls == [[".bilibili.com"]]+ assert extracted == {+ "bilibili": {"SESSDATA": "session", "bili_jct": "csrf"}+ }+++def test_xueqiu_collects_only_xq_a_token(monkeypatch):+ def chrome(domains):+ assert domains == [".xueqiu.com"]+ return [+ {"name": "xq_a_token", "value": "needed", "domain": ".xueqiu.com"},+ {"name": "device_id", "value": "unrelated", "domain": ".xueqiu.com"},+ {"name": "remember", "value": "private", "domain": ".xueqiu.com"},+ ]++ fake_rookiepy = SimpleNamespace(+ chrome=chrome,+ firefox=lambda domains: [],+ edge=lambda domains: [],+ brave=lambda domains: [],+ opera=lambda domains: [],+ )+ monkeypatch.setitem(sys.modules, "rookiepy", fake_rookiepy)++ extracted = cookie_extract.extract_all("chrome", platform="xueqiu")++ assert extracted == {"xueqiu": {"xq_a_token": "needed"}}+++def test_cookie_backend_cannot_smuggle_a_lookalike_domain(monkeypatch):+ def chrome(domains):+ assert domains == [".xueqiu.com"]+ return [+ {"name": "xq_a_token", "value": "valid", "domain": ".xueqiu.com"},+ {+ "name": "xq_a_token",+ "value": "lookalike",+ "domain": ".notxueqiu.com",+ },+ ]++ fake_rookiepy = SimpleNamespace(+ chrome=chrome,+ firefox=lambda domains: [],+ edge=lambda domains: [],+ brave=lambda domains: [],+ opera=lambda domains: [],+ )+ monkeypatch.setitem(sys.modules, "rookiepy", fake_rookiepy)++ extracted = cookie_extract.extract_all("chrome", platform="xueqiu")++ assert extracted == {"xueqiu": {"xq_a_token": "valid"}}+++ ("platform", "manual_key"),+ [+ ("twitter", "twitter-cookies"),+ ("xhs", "xhs-cookies"),+ ],+)+def test_browser_extraction_rejects_cookie_editor_platforms(platform, manual_key):+ with pytest.raises(ValueError, match=rf"Cookie-Editor.*{manual_key}"):+ cookie_extract.extract_all("chrome", platform=platform)+++def test_explicit_profile_uses_only_that_cookie_database(tmp_path, monkeypatch):+ cookie_db = tmp_path / "Profile 1" / "Network" / "Cookies"+ cookie_db.parent.mkdir(parents=True)+ cookie_db.write_bytes(b"test database placeholder")+ monkeypatch.setattr(+ cookie_extract, "_chromium_user_data_dir", lambda browser: tmp_path+ )+ monkeypatch.setitem(sys.modules, "rookiepy", None)++ calls = []++ def chrome(*, cookie_file=None, domain_name=""):+ calls.append((cookie_file, domain_name))+ return []++ fake_browser_cookie3 = SimpleNamespace(+ chrome=chrome,+ firefox=lambda **kwargs: [],+ edge=lambda **kwargs: [],+ brave=lambda **kwargs: [],+ opera=lambda **kwargs: [],+ )+ monkeypatch.setitem(sys.modules, "browser_cookie3", fake_browser_cookie3)++ cookie_extract.extract_all(+ "chrome", platform="xueqiu", profile="Profile 1"+ )++ assert calls == [+ (str(cookie_db), ".xueqiu.com"),+ ]+++def test_missing_explicit_profile_fails_instead_of_falling_back(tmp_path, monkeypatch):+ default_db = tmp_path / "Default" / "Network" / "Cookies"+ default_db.parent.mkdir(parents=True)+ default_db.write_bytes(b"default profile")+ monkeypatch.setattr(+ cookie_extract, "_chromium_user_data_dir", lambda browser: tmp_path+ )++ with pytest.raises(ValueError, match="Profile 7.*not found"):+ cookie_extract.extract_all(+ "chrome", platform="xueqiu", profile="Profile 7"+ )+++def test_configure_requires_platform_before_reading_browser():+ with pytest.raises(ValueError, match="platform"):+ cookie_extract.configure_from_browser("chrome", RecordingConfig())+++def test_invalid_platform_error_does_not_echo_url_secrets():+ with pytest.raises(ValueError) as error:+ cookie_extract.extract_all(+ "chrome",+ platform="https://user:[email protected]/?access_token=secret",+ )++ message = str(error.value)+ assert "user:pass" not in message+ assert "secret" not in message+ assert "***" in message++[email protected]("platform", ["twitter", "xhs"])+def test_configure_from_browser_rejects_cookie_editor_platforms(platform):+ with pytest.raises(ValueError, match="Cookie-Editor"):+ cookie_extract.configure_from_browser(+ "chrome", RecordingConfig(), platform=platform+ )+++def test_xueqiu_config_persists_only_xq_a_token(monkeypatch):+ monkeypatch.setattr(+ cookie_extract,+ "extract_all",+ lambda browser, **kwargs: {"xueqiu": {"xq_a_token": "needed"}},+ )+ config = RecordingConfig()++ result = cookie_extract.configure_from_browser(+ "chrome", config, platform="xueqiu"+ )++ assert config.values == {"xueqiu_cookie": "xq_a_token=needed"}+ assert result[0].targets == ("xueqiu_cookie",)+++def test_bilibili_config_reports_each_written_key(monkeypatch):+ monkeypatch.setattr(+ cookie_extract,+ "extract_all",+ lambda browser, **kwargs: {+ "bilibili": {"SESSDATA": "session", "bili_jct": "csrf"}+ },+ )+ config = RecordingConfig()++ result = cookie_extract.configure_from_browser(+ "chrome", config, platform="bilibili"+ )++ assert config.values == {+ "bilibili_sessdata": "session",+ "bilibili_csrf": "csrf",+ }+ assert result[0].targets == ("bilibili_sessdata", "bilibili_csrf")+ assert tuple(result[0]) == ("Bilibili", True, "SESSDATA + bili_jct")diff --git a/tests/test_scrub_credentials.py b/tests/test_scrub_credentials.pynew file mode 100644index 00000000..2e669ae5--- /dev/null+++ b/tests/test_scrub_credentials.py@@ -0,0 +1,103 @@+"""Secrets embedded in URLs must not reach user-facing diagnostics."""++import sys+from types import SimpleNamespace++import pytest++from agent_reach import cookie_extract+from agent_reach.channels import v2ex as v2ex_module+from agent_reach.channels import xueqiu as xueqiu_module+from agent_reach.channels.v2ex import V2EXChannel+from agent_reach.channels.xueqiu import XueqiuChannel+from agent_reach.utils.text import scrub_url_credentials+++def test_scrubs_userinfo_and_sensitive_query_values():+ raw = (+ "proxy http://user:[email protected]:8080 failed; "+ "upstream https://api.example.test/path?access_token=secret"+ "&page=2&api_key=another-secret"+ )++ scrubbed = scrub_url_credentials(raw)++ assert "user:pass" not in scrubbed+ assert "secret" not in scrubbed+ assert "another-secret" not in scrubbed+ assert "http://***@proxy.example:8080" in scrubbed+ assert "access_token=***" in scrubbed+ assert "page=2" in scrubbed+ assert "api_key=***" in scrubbed+++def test_scrubs_multiple_schemes_and_fragment_tokens():+ raw = (+ "socks5://token@host:1080 "+ "https://example.test/#auth_token=fragment-secret"+ )++ scrubbed = scrub_url_credentials(ValueError(raw))++ assert scrubbed == (+ "socks5://***@host:1080 "+ "https://example.test/#auth_token=***"+ )+++def test_scrubs_bare_user_password_host_diagnostics():+ raw = "proxy handshake for user:[email protected] failed"+ assert scrub_url_credentials(raw) == "proxy handshake for ***@proxy.test failed"+++def test_leaves_non_secret_urls_and_plain_text_unchanged():+ raw = "See https://example.test/search?q=python&page=2 after timeout"+ assert scrub_url_credentials(raw) == raw+++ ("module", "channel"),+ [+ (v2ex_module, V2EXChannel()),+ (xueqiu_module, XueqiuChannel()),+ ],+)+def test_channel_health_messages_scrub_url_secrets(module, channel, monkeypatch):+ def fail(_url):+ raise RuntimeError(+ "proxy http://user:[email protected]:8080 refused "+ "https://api.test/data?access_token=top-secret"+ )++ monkeypatch.setattr(module, "_get_json", fail)++ _status, message = channel.check()++ assert "user:pass" not in message+ assert "top-secret" not in message+ assert "***" in message+++def test_browser_backend_errors_scrub_url_secrets(monkeypatch):+ def chrome(_domains):+ raise RuntimeError(+ "failed via http://user:[email protected] "+ "https://api.test/?token=top-secret"+ )++ fake_rookiepy = SimpleNamespace(+ chrome=chrome,+ firefox=lambda domains: [],+ edge=lambda domains: [],+ brave=lambda domains: [],+ opera=lambda domains: [],+ )+ monkeypatch.setitem(sys.modules, "rookiepy", fake_rookiepy)++ with pytest.raises(RuntimeError) as error:+ cookie_extract.extract_all("chrome", platform="xueqiu")++ message = str(error.value)+ assert "user:pass" not in message+ assert "top-secret" not in message+ assert "***" in messagediff --git a/tests/test_url_security.py b/tests/test_url_security.pynew file mode 100644index 00000000..c8330bd7--- /dev/null+++ b/tests/test_url_security.py@@ -0,0 +1,125 @@+"""Credential-bearing channels must reject lookalike or disguised hosts."""++import pytest++from agent_reach.channels.bilibili import BilibiliChannel+from agent_reach.channels.facebook import FacebookChannel+from agent_reach.channels.github import GitHubChannel+from agent_reach.channels.instagram import InstagramChannel+from agent_reach.channels.linkedin import LinkedInChannel+from agent_reach.channels.reddit import RedditChannel+from agent_reach.channels.twitter import TwitterChannel+from agent_reach.channels.v2ex import V2EXChannel+from agent_reach.channels.xiaohongshu import XiaoHongShuChannel+from agent_reach.channels.xiaoyuzhou import XiaoyuzhouChannel+from agent_reach.channels.xueqiu import XueqiuChannel+from agent_reach.channels.youtube import YouTubeChannel+++ ("channel", "valid_url"),+ [+ (TwitterChannel(), "https://mobile.twitter.com/user/status/1"),+ (TwitterChannel(), "https://X.COM./user/status/1"),+ (XiaoHongShuChannel(), "https://www.xiaohongshu.com/explore/1"),+ (XiaoHongShuChannel(), "https://xhslink.com/a/1"),+ (BilibiliChannel(), "https://www.bilibili.com/video/BV1"),+ (BilibiliChannel(), "https://b23.tv/abc"),+ (XueqiuChannel(), "https://stock.xueqiu.com/v5/stock/quote"),+ ],+)+def test_credential_channels_accept_exact_hosts_and_subdomains(channel, valid_url):+ assert channel.can_handle(valid_url)+++ ("channel", "malicious_url"),+ [+ (TwitterChannel(), "https://x.com.evil.test/user/status/1"),+ (TwitterChannel(), "https://notx.com/user/status/1"),+ (TwitterChannel(), "https://[email protected]/user/status/1"),+ (TwitterChannel(), "https://user:[email protected]/user/status/1"),+ (TwitterChannel(), "ftp://x.com/user/status/1"),+ (XiaoHongShuChannel(), "https://xiaohongshu.com.evil.test/explore/1"),+ (XiaoHongShuChannel(), "https://[email protected]/explore/1"),+ (BilibiliChannel(), "https://bilibili.com.evil.test/video/BV1"),+ (BilibiliChannel(), "https://[email protected]/abc"),+ (XueqiuChannel(), "https://xueqiu.com.evil.test/S/SH600519"),+ (XueqiuChannel(), "https://[email protected]/S/SH600519"),+ ],+)+def test_credential_channels_reject_lookalikes_and_userinfo(channel, malicious_url):+ assert not channel.can_handle(malicious_url)+++ ("channel", "subdomain_url", "port_url"),+ [+ (+ GitHubChannel(),+ "https://api.github.com/repos/openai/openai-python",+ "https://github.com:443/openai/openai-python",+ ),+ (+ YouTubeChannel(),+ "https://m.youtube.com/watch?v=abc",+ "https://youtu.be:443/abc",+ ),+ (+ RedditChannel(),+ "https://old.reddit.com/r/python",+ "https://reddit.com:443/r/python",+ ),+ (+ LinkedInChannel(),+ "https://www.linkedin.com/in/example",+ "https://linkedin.com:443/in/example",+ ),+ (+ V2EXChannel(),+ "https://www.v2ex.com/t/1",+ "https://v2ex.com:443/t/1",+ ),+ (+ XiaoyuzhouChannel(),+ "https://www.xiaoyuzhoufm.com/episode/1",+ "https://xiaoyuzhoufm.com:443/episode/1",+ ),+ (+ FacebookChannel(),+ "https://m.facebook.com/groups/1",+ "https://facebook.com:443/groups/1",+ ),+ (+ InstagramChannel(),+ "https://www.instagram.com/example",+ "https://instagram.com:443/example",+ ),+ ],+)+def test_fixed_domain_channels_accept_subdomains_and_explicit_ports(+ channel, subdomain_url, port_url+):+ assert channel.can_handle(subdomain_url)+ assert channel.can_handle(port_url)+++ ("channel", "official_domain"),+ [+ (GitHubChannel(), "github.com"),+ (YouTubeChannel(), "youtube.com"),+ (RedditChannel(), "reddit.com"),+ (LinkedInChannel(), "linkedin.com"),+ (V2EXChannel(), "v2ex.com"),+ (XiaoyuzhouChannel(), "xiaoyuzhoufm.com"),+ (FacebookChannel(), "facebook.com"),+ (InstagramChannel(), "instagram.com"),+ ],+)+def test_fixed_domain_channels_reject_suffix_lookalikes_and_userinfo(+ channel, official_domain+):+ assert not channel.can_handle(f"https://{official_domain}.evil.test/path")+ assert not channel.can_handle(f"https://{official_domain}@evil.test/path")+ assert not channel.can_handle(f"https://user:pass@{official_domain}/path")