Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.
Python agent-infrastructureai-agentai-searchautomationbilibiliclaude-codeclicursorfree-apillm-toolsmcppythonreddit-scrapertwitter-scraperweb-scraperxiaohongshuyoutube-transcript
feat: move twitter, linkedin, boss, xiaoyuzhou to unified entry
Pnant8fff8910fbaf
diff --git a/agent_reach/channels/base.py b/agent_reach/channels/base.pyindex 22e6ead1..21399812 100644--- a/agent_reach/channels/base.py+++ b/agent_reach/channels/base.py@@ -53,6 +53,8 @@ class Action:params: Tuple[Param, ...]backends: Tuple[Tuple[str, str], ...]live_test: Optional[Dict[str, object]] = None+ #: Default timeout in seconds for slow actions (e.g. transcription); None = global default.+ timeout: Optional[float] = Noneclass Channel(ABC):diff --git a/agent_reach/channels/boss.py b/agent_reach/channels/boss.pyindex 4c78629a..cf54eefb 100644--- a/agent_reach/channels/boss.py+++ b/agent_reach/channels/boss.py@@ -35,9 +35,17 @@ import urllib.requestfrom urllib.parse import urlparsefrom agent_reach.probe import probe_command+from agent_reach.runtime.result import (+ NEED_LOGIN,+ NEED_SETUP,+ UPSTREAM_BROKEN,+ ReachError,+ make_item,+)+from agent_reach.runtime.run import CommandFailed, run_cmdfrom agent_reach.utils.url import host_matches-from .base import Channel+from .base import Action, Channel, Param_CDP_URL = "http://localhost:9222"_CDP_TIMEOUT = 5@@ -256,6 +264,22 @@ class BossChannel(Channel):backends = ["boss-agent-cli (CDP)"]tier = 2+ level = "optional"+ actions = (+ Action(+ "search",+ "Search jobs (needs the dedicated Chrome with a zhipin.com login)",+ (+ Param("query", "job keywords"),+ Param("city", "city, e.g. 深圳", required=False),+ Param("limit", "max results", required=False, type=int, default=10),+ ),+ (("boss-agent-cli", "_cli_search"),),+ live_test={"query": "Python", "city": "深圳", "limit": 3},+ timeout=90,+ ),+ )+def can_handle(self, url: str) -> bool:return host_matches(url, "zhipin.com")@@ -334,3 +358,56 @@ class BossChannel(Channel):"若搜索报 AUTH_EXPIRED,按登录 runbook 处理(用户在专用窗口登录 + login --cdp),""不要往安全校验方向解释。")++ # ── unified entry backends ──++ def _cli_search(self, *, query: str, city, limit: int, timeout: float) -> list:+ with socket.socket() as s:+ s.settimeout(1)+ cdp_up = s.connect_ex(("127.0.0.1", 9222)) == 0+ if not cdp_up:+ raise ReachError(+ NEED_SETUP,+ "the dedicated Chrome (port 9222) is not running",+ hint=f"Ask the user to start it with: {_chrome_launch_command()} and log in to zhipin.com themselves.",+ )+ args = ["boss", "--json", "--browser-source", "existing-browser", "--cdp-url", _CDP_URL, "search", query]+ if city:+ args += ["--city", city]+ try:+ out = run_cmd(args, timeout=timeout)+ except CommandFailed as exc:+ out = exc.stdout+ if "{" not in out:+ raise+ try:+ payload = json.loads(out[out.index("{"):])+ except ValueError:+ raise ReachError(UPSTREAM_BROKEN, "boss-agent-cli returned unreadable output") from None+ if not payload.get("ok"):+ err = payload.get("error") or {}+ code = str(err.get("code", "")).upper()+ message = str(err.get("message", "boss-agent-cli failed"))[:300]+ if "AUTH" in code or "LOGIN" in code:+ raise ReachError(+ NEED_LOGIN, message,+ hint="Ask the user to log in to zhipin.com in the dedicated Chrome window, then run `boss --cdp-url http://localhost:9222 login --cdp`.",+ )+ raise ReachError(UPSTREAM_BROKEN, message)+ data = payload.get("data")+ rows = data if isinstance(data, list) else ((data or {}).get("items") or (data or {}).get("jobs") or [])+ items = []+ for r in rows[:limit]:+ if not isinstance(r, dict):+ continue+ job_id = r.get("job_id")+ items.append(+ make_item(+ title=r.get("title"),+ url=f"https://www.zhipin.com/job_detail/{job_id}.html" if job_id else None,+ author=r.get("company"),+ text=" · ".join(str(x) for x in (r.get("salary"), r.get("city"), r.get("experience"), r.get("education")) if x),+ raw=r,+ )+ )+ return itemsdiff --git a/agent_reach/channels/linkedin.py b/agent_reach/channels/linkedin.pyindex b3a535e2..8c9cfe42 100644--- a/agent_reach/channels/linkedin.py+++ b/agent_reach/channels/linkedin.py@@ -1,10 +1,14 @@# -*- coding: utf-8 -*-"""LinkedIn — check if mcp-server-linkedin is configured."""+import jsonimport shutil-from .base import Channel-from .mcporter import McporterConfigError, inspect_mcporter_config+from agent_reach.runtime.result import NEED_SETUP, UPSTREAM_BROKEN, ReachError, make_item+from agent_reach.runtime.run import CommandFailed, run_cmd++from .base import Action, Channel, Param+from .mcporter import McporterConfigError, configured_server_names, inspect_mcporter_config_LINKEDIN_SERVER_NAMES = {"linkedin",@@ -26,6 +30,22 @@ class LinkedInChannel(Channel):backends = ["mcp-server-linkedin", "Jina Reader"]tier = 2+ level = "optional"+ actions = (+ Action(+ "jobs",+ "Search LinkedIn jobs",+ (+ Param("query", "job keywords"),+ Param("location", "city or country", required=False),+ Param("limit", "max results", required=False, type=int, default=5),+ ),+ (("linkedin-mcp", "_mcp_jobs"),),+ live_test={"query": "python", "limit": 2},+ timeout=120,+ ),+ )+def can_handle(self, url: str) -> bool:from agent_reach.utils.url import host_matches@@ -67,3 +87,39 @@ class LinkedInChannel(Channel):f" {_LOGIN_COMMAND}\n"f" {_CONFIG_COMMAND}")++ # ── unified entry backends ──++ def _mcp_jobs(self, *, query: str, location, limit: int, timeout: float) -> list:+ out = run_cmd(["mcporter", "config", "list", "--json"], timeout=min(timeout, 20))+ names = configured_server_names(out)+ server = next((n for n in ("linkedin", "mcp-server-linkedin", "linkedin-scraper", "linkedin-scraper-mcp") if n in names), None)+ if server is None:+ raise ReachError(NEED_SETUP, "no LinkedIn MCP server is registered in mcporter", hint=f"Set up: {_CONFIG_COMMAND}")+ # The official server names the query `keywords`; the scraper variants use `query`.+ args = {"keywords": query} if server in ("linkedin", "mcp-server-linkedin") else {"query": query, "max_results": limit}+ if location:+ args["location"] = location+ try:+ raw = run_cmd(+ ["mcporter", "call", f"{server}.search_jobs", "--args", json.dumps(args), "--output", "json"],+ timeout=timeout,+ )+ except CommandFailed as exc:+ raise ReachError(UPSTREAM_BROKEN, exc.message) from None+ try:+ data = json.loads(raw)+ except ValueError:+ raise ReachError(UPSTREAM_BROKEN, "LinkedIn MCP returned unreadable output") from None+ rows = data if isinstance(data, list) else (data.get("results") or data.get("jobs") or [])+ return [+ make_item(+ title=r.get("title"),+ url=r.get("job_url") or r.get("url"),+ author=r.get("company"),+ text=" · ".join(x for x in (r.get("company"), r.get("location")) if x),+ raw=r,+ )+ for r in rows[:limit]+ if isinstance(r, dict)+ ]diff --git a/agent_reach/channels/twitter.py b/agent_reach/channels/twitter.pyindex 006a85b2..2e66a58b 100644--- a/agent_reach/channels/twitter.py+++ b/agent_reach/channels/twitter.py@@ -4,9 +4,23 @@import osimport shutil+from agent_reach.backends.opencli import opencli_call, opencli_items+from agent_reach.runtime.result import UPSTREAM_BROKEN, ReachErrorfrom agent_reach.utils.url import host_matches-from .base import Channel+from .base import Action, Channel, Param++_TWITTER_LOGIN_HINT = (+ "Ask the user to log in to x.com in their own Chrome (the one with the "+ "OpenCLI extension). Never log in or read browser cookies for them."+)+++def _x(args, timeout: float) -> list:+ items = opencli_items(opencli_call("twitter", list(args), timeout=timeout, login_hint=_TWITTER_LOGIN_HINT))+ if not items:+ raise ReachError(UPSTREAM_BROKEN, "OpenCLI returned nothing; the page most likely did not load")+ return itemsdef twitter_cli_child_env(config=None) -> dict[str, str]:@@ -37,6 +51,44 @@ class TwitterChannel(Channel):backends = ["twitter-cli", "OpenCLI", "bird CLI (legacy)"]tier = 1+ # twitter-cli is not used here: even with explicit credentials it re-reads+ # browser cookies whenever verification fails.+ level = "experimental"+ actions = (+ Action(+ "search",+ "Search tweets",+ (+ Param("query", "what to search for"),+ Param("limit", "max results", required=False, type=int, default=5),+ ),+ (("opencli", "_opencli_search"),),+ live_test={"query": "openai", "limit": 2},+ ),+ Action(+ "profile",+ "A user's profile",+ (Param("username", "handle without @"),),+ (("opencli", "_opencli_profile"),),+ live_test={"username": "OpenAI"},+ ),+ Action(+ "tweets",+ "A user's recent tweets",+ (+ Param("username", "handle without @"),+ Param("limit", "max tweets", required=False, type=int, default=5),+ ),+ (("opencli", "_opencli_tweets"),),+ ),+ Action(+ "thread",+ "A tweet with its replies",+ (Param("target", "tweet id or link"),),+ (("opencli", "_opencli_thread"),),+ ),+ )+def can_handle(self, url: str) -> bool:return host_matches(url, "x.com", "twitter.com")@@ -140,3 +192,18 @@ class TwitterChannel(Channel):"仅使用 Cookie-Editor 手动导出的凭据。")return None++ # ── unified entry backends ──++ def _opencli_search(self, *, query: str, limit: int, timeout: float) -> list:+ return _x(["search", query, "--limit", str(limit)], timeout)[:limit]++ def _opencli_profile(self, *, username: str, timeout: float) -> list:+ return _x(["profile", username.lstrip("@")], timeout)[:1]++ def _opencli_tweets(self, *, username: str, limit: int, timeout: float) -> list:+ return _x(["tweets", username.lstrip("@"), "--limit", str(limit)], timeout)[:limit]++ def _opencli_thread(self, *, target: str, timeout: float) -> list:+ tweet_id = str(target).rstrip("/").split("/")[-1].split("?")[0]+ return _x(["thread", tweet_id], timeout)diff --git a/agent_reach/channels/xiaoyuzhou.py b/agent_reach/channels/xiaoyuzhou.pyindex 5afee0e1..24db14d9 100644--- a/agent_reach/channels/xiaoyuzhou.py+++ b/agent_reach/channels/xiaoyuzhou.py@@ -2,11 +2,17 @@"""Xiaoyuzhou Podcast (小宇宙播客) — transcribe podcasts via Groq Whisper API."""import os+import shutil+import tempfilefrom agent_reach.config import Configfrom agent_reach.probe import probe_command+from agent_reach.runtime.result import BAD_INPUT, NEED_SETUP, ReachError, make_item+from agent_reach.runtime.run import run_cmd-from .base import Channel+from .base import Action, Channel, Param++_SCRIPT = os.path.join(os.path.dirname(os.path.dirname(__file__)), "scripts", "transcribe_xiaoyuzhou.sh")class XiaoyuzhouChannel(Channel):@@ -15,6 +21,17 @@ class XiaoyuzhouChannel(Channel):backends = ["groq-whisper", "ffmpeg"]tier = 1+ level = "optional"+ actions = (+ Action(+ "transcribe",+ "Full transcript of a podcast episode (uses Groq Whisper quota)",+ (Param("target", "episode link"),),+ (("groq-whisper", "_script_transcribe"),),+ timeout=900,+ ),+ )+def can_handle(self, url: str) -> bool:from agent_reach.utils.url import host_matches@@ -63,3 +80,25 @@ class XiaoyuzhouChannel(Channel):self.active_backend = "groq-whisper"return "ok", "完整可用(播客下载 + Whisper 转录)"++ # ── unified entry backends ──++ def _script_transcribe(self, *, target: str, timeout: float) -> list:+ from agent_reach.utils.url import host_matches++ if not host_matches(str(target).strip(), "xiaoyuzhoufm.com"):+ raise ReachError(BAD_INPUT, "target must be a xiaoyuzhoufm.com episode link")+ key = os.environ.get("GROQ_API_KEY") or Config(read_only=True).get("groq_api_key")+ if not key:+ raise ReachError(+ NEED_SETUP, "no Groq API key configured",+ hint="Ask the user for a free key from https://console.groq.com, then run `agent-reach configure groq-key`.",+ )+ if not shutil.which("ffmpeg"):+ raise ReachError(NEED_SETUP, "ffmpeg is not installed", hint="Ask the user to install ffmpeg (macOS: brew install ffmpeg).")+ with tempfile.TemporaryDirectory(prefix="agent-reach-xyz-") as tmp:+ out = os.path.join(tmp, "transcript.txt")+ run_cmd(["bash", _SCRIPT, str(target).strip(), out], timeout=timeout, env={"GROQ_API_KEY": str(key)}, label="transcribe")+ with open(out, encoding="utf-8", errors="replace") as fh:+ text = fh.read().strip()+ return [make_item(url=str(target).strip(), text=text)]diff --git a/agent_reach/runtime/cli.py b/agent_reach/runtime/cli.pyindex 13a7d72d..0ce8f6ad 100644--- a/agent_reach/runtime/cli.py+++ b/agent_reach/runtime/cli.py@@ -73,7 +73,7 @@ def _parse_action_args(action: Action, args: Sequence[str]) -> Dict[str, object]parser.add_argument(p.name, nargs="+" if p.type is str else None)else:parser.add_argument(f"--{p.name}", dest=p.name, default=None)- parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT)+ parser.add_argument("--timeout", type=float, default=action.timeout or DEFAULT_TIMEOUT)ns = vars(parser.parse_args(list(args)))for p in action.params:if p.required and p.type is str and isinstance(ns.get(p.name), list):