Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.
Python agent-infrastructureai-agentai-searchautomationbilibiliclaude-codeclicursorfree-apillm-toolsmcppythonreddit-scrapertwitter-scraperweb-scraperxiaohongshuyoutube-transcript
feat: move xueqiu and bilibili to unified entry
Pnant8ebd2fbbf0f8
diff --git a/agent_reach/channels/bilibili.py b/agent_reach/channels/bilibili.pyindex 5c500585..5b0cab0f 100644--- a/agent_reach/channels/bilibili.py+++ b/agent_reach/channels/bilibili.py@@ -10,11 +10,109 @@ YouTube backend; it just no longer serves bilibili."""import json+import os+import reimport urllib.requestfrom agent_reach.probe import probe_command+from agent_reach.runtime.result import (+ BAD_INPUT,+ NEED_LOGIN,+ NOT_FOUND,+ UPSTREAM_BROKEN,+ ReachError,+ make_item,+ unix_to_iso,+)+from agent_reach.runtime.run import CommandFailed, http_request, run_cmd++from .base import Action, Channel, Param++_BV_RE = re.compile(r"BV[0-9A-Za-z]{10}")+_BILI_LOGIN_HINT = (+ "Bilibili subtitles need a login. Ask the user to run `bili login` and scan "+ "the QR code themselves; never log in on their behalf."+)+++def _bvid(target: str, timeout: float) -> str:+ """BV id from a BV id, a video link, or a b23.tv short link."""+ t = str(target).strip()+ m = _BV_RE.search(t)+ if m:+ return m.group(0)+ if "b23.tv/" in t:+ url = t if "://" in t else f"https://{t}"+ req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": _UA})+ with urllib.request.urlopen(req, timeout=timeout) as resp:+ m = _BV_RE.search(resp.geturl())+ if m:+ return m.group(0)+ raise ReachError(BAD_INPUT, "target must be a BV id or a bilibili.com / b23.tv video link")+++def _bili_json(args: list, timeout: float) -> dict:+ """Run bili-cli with --json; map its error envelope to unified codes."""+ try:+ out = run_cmd(["bili", *args, "--json"], timeout=timeout)+ except CommandFailed as exc:+ out = exc.stdout+ if "{" not in out:+ raise+ try:+ payload = json.loads(out[out.index("{"):])+ except ValueError:+ raise ReachError(UPSTREAM_BROKEN, "bili-cli returned unreadable output") from None+ if not payload.get("ok"):+ err = payload.get("error") or {}+ code = str(err.get("code", ""))+ message = str(err.get("message", "bili-cli failed"))[:300]+ if code == "not_found":+ raise ReachError(NOT_FOUND, message)+ if "login" in code or "auth" in code or "credential" in code:+ raise ReachError(NEED_LOGIN, message, hint=_BILI_LOGIN_HINT)+ raise ReachError(UPSTREAM_BROKEN, message)+ return payload.get("data") or {}+++def _cli_video_item(v: dict, text=None) -> dict:+ owner = v.get("owner") or {}+ return make_item(+ title=v.get("title"),+ url=v.get("url") or (f"https://www.bilibili.com/video/{v['bvid']}" if v.get("bvid") else None),+ author=owner.get("name") or v.get("author"),+ text=v.get("description") if text is None else text,+ raw=v,+ )+++def _api_video_item(v: dict, text=None) -> dict:+ return make_item(+ title=v.get("title"),+ url=f"https://www.bilibili.com/video/{v.get('bvid')}" if v.get("bvid") else None,+ author=(v.get("owner") or {}).get("name"),+ published_at=unix_to_iso(v.get("pubdate")),+ text=v.get("desc") if text is None else text,+ raw={k: v.get(k) for k in ("bvid", "aid", "duration", "stat", "tname")},+ )+++def _bili_api(path: str, timeout: float) -> dict:+ body = http_request(+ f"https://api.bilibili.com{path}",+ timeout=timeout,+ headers={"User-Agent": _UA, "Referer": "https://www.bilibili.com/"},+ )+ try:+ data = json.loads(body)+ except ValueError:+ raise ReachError(UPSTREAM_BROKEN, "Bilibili returned a verification page") from None+ if data.get("code") == -404:+ raise ReachError(NOT_FOUND, "video not found")+ if data.get("code") != 0:+ raise ReachError(UPSTREAM_BROKEN, f"Bilibili API code {data.get('code')}: {data.get('message')}")+ return data.get("data") or {}-from .base import Channel_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"_TIMEOUT = 10@@ -38,6 +136,41 @@ class BilibiliChannel(Channel):backends = ["bili-cli", "OpenCLI", "B站搜索 API"]tier = 1+ level = "optional"+ actions = (+ Action(+ "search",+ "Search videos",+ (+ Param("query", "what to search for"),+ Param("limit", "max results", required=False, type=int, default=5),+ ),+ (("bili-cli", "_cli_search"),),+ live_test={"query": "Python", "limit": 2},+ ),+ Action(+ "read",+ "Video title, uploader, description and stats",+ (Param("target", "BV id or video link"),),+ (("bili-cli", "_cli_read"), ("bilibili-api", "_api_read")),+ live_test={"target": "BV1GJ411x7h7"},+ ),+ Action(+ "hot",+ "Trending videos",+ (Param("limit", "max items", required=False, type=int, default=10),),+ (("bili-cli", "_cli_hot"), ("bilibili-api", "_api_hot")),+ live_test={"limit": 2},+ ),+ Action(+ "subtitle",+ "Full subtitle text of a video (needs a Bilibili login)",+ (Param("target", "BV id or video link"),),+ (("bili-cli", "_cli_subtitle"),),+ live_test={"target": "BV1GJ411x7h7"},+ ),+ )+def can_handle(self, url: str) -> bool:from agent_reach.utils.url import host_matches@@ -117,3 +250,34 @@ class BilibiliChannel(Channel):"B站搜索 API 可达(仅搜索,curl 直连)。""完整功能建议安装 bili-cli:pipx install bilibili-cli")++ # ── unified entry backends ──++ def _cli_search(self, *, query: str, limit: int, timeout: float) -> list:+ data = _bili_json(["search", query, "--type", "video", "-n", str(limit)], timeout)+ return [_cli_video_item(v) for v in (data if isinstance(data, list) else [])[:limit]]++ def _cli_read(self, *, target: str, timeout: float) -> list:+ data = _bili_json(["video", _bvid(target, timeout)], timeout)+ return [_cli_video_item(data.get("video") or {})]++ def _api_read(self, *, target: str, timeout: float) -> list:+ return [_api_video_item(_bili_api(f"/x/web-interface/view?bvid={_bvid(target, timeout)}", timeout))]++ def _cli_hot(self, *, limit: int, timeout: float) -> list:+ data = _bili_json(["hot", "-n", str(limit)], timeout)+ return [_cli_video_item(v) for v in (data.get("items") or [])[:limit]]++ def _api_hot(self, *, limit: int, timeout: float) -> list:+ data = _bili_api(f"/x/web-interface/popular?ps={limit}&pn=1", timeout)+ return [_api_video_item(v) for v in (data.get("list") or [])[:limit]]++ def _cli_subtitle(self, *, target: str, timeout: float) -> list:+ data = _bili_json(["video", _bvid(target, timeout), "--subtitle"], timeout)+ sub = data.get("subtitle") or {}+ if not sub.get("available"):+ if not os.path.exists(os.path.expanduser("~/.bilibili-cli/credential.json")):+ raise ReachError(NEED_LOGIN, "Bilibili is not logged in", hint=_BILI_LOGIN_HINT)+ raise ReachError(NOT_FOUND, "this video has no subtitles")+ text = sub.get("text") or "\n".join(i.get("content", "") for i in sub.get("items") or [])+ return [_cli_video_item(data.get("video") or {}, text=text)]diff --git a/agent_reach/channels/xueqiu.py b/agent_reach/channels/xueqiu.pyindex 1ecf45b5..765837b5 100644--- a/agent_reach/channels/xueqiu.py+++ b/agent_reach/channels/xueqiu.py@@ -4,11 +4,21 @@import http.cookiejarimport jsonimport re+import urllib.errorimport urllib.parseimport urllib.requestfrom typing import Any-from .base import Channel+from agent_reach.runtime.result import BAD_INPUT, NEED_LOGIN, NOT_FOUND, ReachError, make_item+from agent_reach.runtime.run import http_request++from .base import Action, Channel, Param++_XQ_LOGIN_HINT = (+ "Xueqiu community data needs a browser session. Ask the user to open "+ "https://xueqiu.com in Chrome, then run `agent-reach configure --from-browser "+ "chrome --platform xueqiu`. The session expires within hours."+)_UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "@@ -111,12 +121,115 @@ def _strip_html(text: str) -> str:return text.strip()+def _tencent_code(symbol: str) -> str:+ """SH600519 / 600519 / HK00700 / 00700 / AAPL → Tencent quote code."""+ s = str(symbol).strip().upper()+ if re.fullmatch(r"(SH|SZ|BJ)\d{6}", s):+ return s.lower()+ if re.fullmatch(r"\d{6}", s):+ prefix = "sh" if s[0] in "69" else ("bj" if s[0] in "48" else "sz")+ return prefix + s+ if re.fullmatch(r"HK\d{1,5}", s):+ return "hk" + s[2:].zfill(5)+ if re.fullmatch(r"\d{1,5}", s):+ return "hk" + s.zfill(5)+ if re.fullmatch(r"[A-Z][A-Z.\-]{0,9}", s):+ return "us" + s+ raise ReachError(BAD_INPUT, "symbol must look like SH600519, 600519, 00700 or AAPL")+++def _xueqiu_symbol(market: str, code: str) -> str:+ """Tencent market + code → the symbol Xueqiu uses in URLs and its API."""+ market = market.lower()+ if market in ("sh", "sz", "bj"):+ return market.upper() + code+ if market == "hk":+ return code+ return code.split(".")[0].upper()+++def _xq_json(url: str) -> Any:+ try:+ return _get_json(url)+ except urllib.error.HTTPError as exc:+ if exc.code in (400, 401, 403):+ raise ReachError(NEED_LOGIN, f"Xueqiu rejected the session (HTTP {exc.code})", hint=_XQ_LOGIN_HINT) from None+ raise+ except json.JSONDecodeError:+ raise ReachError(NEED_LOGIN, "Xueqiu returned a verification page instead of data", hint=_XQ_LOGIN_HINT) from None+++_HOT_POSTS_URL = (+ "https://xueqiu.com/v4/statuses/public_timeline_by_category.json"+ "?since_id=-1&max_id=-1&count={limit}&category=-1"+)+++def _parse_hot_posts(data: Any, limit: int) -> list:+ """Each list item carries a JSON-encoded `data` field with the real post."""+ results = []+ for item in (data.get("list") or [])[:limit]:+ try:+ post = json.loads(item["data"]) if isinstance(item.get("data"), str) else {}+ except (json.JSONDecodeError, KeyError):+ post = {}+ user = post.get("user") or {}+ text = _strip_html(post.get("text") or post.get("description") or "")+ target = post.get("target", "")+ results.append(+ {+ "id": post.get("id", 0),+ "title": post.get("title") or "",+ "text": text[:200],+ "author": user.get("screen_name", ""),+ "likes": post.get("like_count", 0),+ "url": f"https://xueqiu.com{target}" if target else "",+ }+ )+ return results++class XueqiuChannel(Channel):name = "xueqiu"description = "雪球股票行情与社区动态"backends = ["Xueqiu API (需要登录 Cookie)"]tier = 1+ level = "optional"+ actions = (+ Action(+ "quote",+ "Real-time stock quote (A-share, HK, US)",+ (Param("symbol", "e.g. SH600519, 600519, 00700, AAPL"),),+ (("tencent", "_tencent_quote"), ("xueqiu-api", "_xq_quote")),+ live_test={"symbol": "SH600519"},+ ),+ Action(+ "search",+ "Find stocks by name or code",+ (+ Param("query", "name or code, e.g. 茅台"),+ Param("limit", "max results", required=False, type=int, default=5),+ ),+ (("tencent", "_tencent_search"), ("xueqiu-api", "_xq_search")),+ live_test={"query": "茅台", "limit": 3},+ ),+ Action(+ "hot_posts",+ "Trending Xueqiu community posts (needs a browser session)",+ (Param("limit", "max items", required=False, type=int, default=10),),+ (("xueqiu-api", "_xq_hot_posts"),),+ live_test={"limit": 3},+ ),+ Action(+ "hot_stocks",+ "Most-watched stocks on Xueqiu (needs a browser session)",+ (Param("limit", "max items", required=False, type=int, default=10),),+ (("xueqiu-api", "_xq_hot_stocks"),),+ live_test={"limit": 3},+ ),+ )+# ------------------------------------------------------------------ ## URL routing# ------------------------------------------------------------------ #@@ -240,38 +353,7 @@ class XueqiuChannel(Channel):limit = min(limit, 50)if limit == 0:return []- data = _get_json(- "https://xueqiu.com/v4/statuses/public_timeline_by_category.json"- f"?since_id=-1&max_id=-1&count={limit}&category=-1"- )- items = data.get("list") or []- results = []- for item in items[:limit]:- # Each item.data is a JSON string containing the real post payload- try:- post = (- json.loads(item["data"])- if isinstance(item.get("data"), str)- else {}- )- except (json.JSONDecodeError, KeyError):- post = {}- user = post.get("user") or {}- text = _strip_html(- post.get("text") or post.get("description") or ""- )- target = post.get("target", "")- results.append(- {- "id": post.get("id", 0),- "title": post.get("title") or "",- "text": text[:200],- "author": user.get("screen_name", ""),- "likes": post.get("like_count", 0),- "url": f"https://xueqiu.com{target}" if target else "",- }- )- return results+ return _parse_hot_posts(_get_json(_HOT_POSTS_URL.format(limit=limit)), limit)def get_hot_stocks(self, limit: int = 10, stock_type: int = 10) -> list:"""获取热门股票排行。@@ -300,3 +382,91 @@ class XueqiuChannel(Channel):})return results++ # ── unified entry backends ──++ def _tencent_quote(self, *, symbol: str, timeout: float) -> list:+ code = _tencent_code(symbol)+ body = http_request(f"https://qt.gtimg.cn/q={code}", timeout=timeout).decode("gbk", errors="replace")+ m = re.search(r'="([^"]*)"', body)+ if not m or m.group(1).count("~") < 34:+ raise ReachError(NOT_FOUND, f"no quote for {symbol}")+ f = m.group(1).split("~")+ xq = _xueqiu_symbol(code[:2], f[2])+ quote = {+ "symbol": xq, "name": f[1], "current": f[3], "last_close": f[4], "open": f[5],+ "chg": f[31], "percent": f[32], "high": f[33], "low": f[34], "time": f[30],+ }+ return [+ make_item(+ title=f"{f[1]} ({xq})",+ url=f"https://xueqiu.com/S/{xq}",+ text=f"{f[1]} {f[3]} ({f[32]}%)",+ raw=quote,+ )+ ]++ def _tencent_search(self, *, query: str, limit: int, timeout: float) -> list:+ q = urllib.parse.quote(query)+ body = http_request(f"https://smartbox.gtimg.cn/s3/?v=2&q={q}&t=all", timeout=timeout).decode("utf-8", errors="replace")+ m = re.search(r'="([^"]*)"', body)+ if not m or m.group(1) in ("", "N"):+ return []+ items = []+ for entry in json.loads(f'"{m.group(1)}"').split("^")[:limit]:+ parts = entry.split("~")+ if len(parts) < 3:+ continue+ market, code, name = parts[0], parts[1], parts[2]+ xq = _xueqiu_symbol(market, code)+ items.append(+ make_item(+ title=f"{name} ({xq})",+ url=f"https://xueqiu.com/S/{xq}",+ raw={"symbol": xq, "name": name, "market": market, "type": parts[4] if len(parts) > 4 else None},+ )+ )+ return items++ def _xq_quote(self, *, symbol: str, timeout: float) -> list:+ code = _tencent_code(symbol)+ xq = _xueqiu_symbol(code[:2], code[2:])+ data = _xq_json(f"https://stock.xueqiu.com/v5/stock/quote.json?symbol={urllib.parse.quote(xq)}&extend=detail")+ q = (data.get("data") or {}).get("quote")+ if not q:+ raise ReachError(NOT_FOUND, f"no quote for {symbol}")+ return [+ make_item(+ title=f"{q.get('name')} ({q.get('symbol')})",+ url=f"https://xueqiu.com/S/{q.get('symbol')}",+ text=f"{q.get('name')} {q.get('current')} ({q.get('percent')}%)",+ raw=q,+ )+ ]++ def _xq_search(self, *, query: str, limit: int, timeout: float) -> list:+ data = _xq_json(f"https://xueqiu.com/stock/search.json?code={urllib.parse.quote(query)}&size={limit}")+ return [+ make_item(title=f"{s.get('name')} ({s.get('code')})", url=f"https://xueqiu.com/S/{s.get('code')}", raw=s)+ for s in (data.get("stocks") or [])[:limit]+ ]++ def _xq_hot_posts(self, *, limit: int, timeout: float) -> list:+ limit = max(0, min(limit, 50))+ posts = _parse_hot_posts(_xq_json(_HOT_POSTS_URL.format(limit=limit)), limit) if limit else []+ return [+ make_item(title=p["title"] or p["text"][:60], url=p["url"] or None, author=p["author"], text=p["text"], raw=p)+ for p in posts+ ]++ def _xq_hot_stocks(self, *, limit: int, timeout: float) -> list:+ data = _xq_json(f"https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size={limit}&type=10")+ return [+ make_item(+ title=f"{s.get('name')} ({s.get('code') or s.get('symbol')})",+ url=f"https://xueqiu.com/S/{s.get('code') or s.get('symbol')}",+ text=f"{s.get('name')} {s.get('current')} ({s.get('percent')}%)",+ raw=s,+ )+ for s in ((data.get("data") or {}).get("items") or [])[:limit]+ ]