Skip to content

Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.

feat: move xueqiu and bilibili to unified entry

Pnant8ebd2fbbf0f8
diff --git a/agent_reach/channels/bilibili.py b/agent_reach/channels/bilibili.py
index 5c500585..5b0cab0f 100644
--- a/agent_reach/channels/bilibili.py
+++ b/agent_reach/channels/bilibili.py
@@ -10,11 +10,109 @@ YouTube backend; it just no longer serves bilibili.
"""
import json
+import os
+import re
import urllib.request
from agent_reach.probe import probe_command
+from agent_reach.runtime.result import (
+ BAD_INPUT,
+ NEED_LOGIN,
+ NOT_FOUND,
+ UPSTREAM_BROKEN,
+ ReachError,
+ make_item,
+ unix_to_iso,
+)
+from agent_reach.runtime.run import CommandFailed, http_request, run_cmd
+
+from .base import Action, Channel, Param
+
+_BV_RE = re.compile(r"BV[0-9A-Za-z]{10}")
+_BILI_LOGIN_HINT = (
+ "Bilibili subtitles need a login. Ask the user to run `bili login` and scan "
+ "the QR code themselves; never log in on their behalf."
+)
+
+
+def _bvid(target: str, timeout: float) -> str:
+ """BV id from a BV id, a video link, or a b23.tv short link."""
+ t = str(target).strip()
+ m = _BV_RE.search(t)
+ if m:
+ return m.group(0)
+ if "b23.tv/" in t:
+ url = t if "://" in t else f"https://{t}"
+ req = urllib.request.Request(url, method="HEAD", headers={"User-Agent": _UA})
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
+ m = _BV_RE.search(resp.geturl())
+ if m:
+ return m.group(0)
+ raise ReachError(BAD_INPUT, "target must be a BV id or a bilibili.com / b23.tv video link")
+
+
+def _bili_json(args: list, timeout: float) -> dict:
+ """Run bili-cli with --json; map its error envelope to unified codes."""
+ try:
+ out = run_cmd(["bili", *args, "--json"], timeout=timeout)
+ except CommandFailed as exc:
+ out = exc.stdout
+ if "{" not in out:
+ raise
+ try:
+ payload = json.loads(out[out.index("{"):])
+ except ValueError:
+ raise ReachError(UPSTREAM_BROKEN, "bili-cli returned unreadable output") from None
+ if not payload.get("ok"):
+ err = payload.get("error") or {}
+ code = str(err.get("code", ""))
+ message = str(err.get("message", "bili-cli failed"))[:300]
+ if code == "not_found":
+ raise ReachError(NOT_FOUND, message)
+ if "login" in code or "auth" in code or "credential" in code:
+ raise ReachError(NEED_LOGIN, message, hint=_BILI_LOGIN_HINT)
+ raise ReachError(UPSTREAM_BROKEN, message)
+ return payload.get("data") or {}
+
+
+def _cli_video_item(v: dict, text=None) -> dict:
+ owner = v.get("owner") or {}
+ return make_item(
+ title=v.get("title"),
+ url=v.get("url") or (f"https://www.bilibili.com/video/{v['bvid']}" if v.get("bvid") else None),
+ author=owner.get("name") or v.get("author"),
+ text=v.get("description") if text is None else text,
+ raw=v,
+ )
+
+
+def _api_video_item(v: dict, text=None) -> dict:
+ return make_item(
+ title=v.get("title"),
+ url=f"https://www.bilibili.com/video/{v.get('bvid')}" if v.get("bvid") else None,
+ author=(v.get("owner") or {}).get("name"),
+ published_at=unix_to_iso(v.get("pubdate")),
+ text=v.get("desc") if text is None else text,
+ raw={k: v.get(k) for k in ("bvid", "aid", "duration", "stat", "tname")},
+ )
+
+
+def _bili_api(path: str, timeout: float) -> dict:
+ body = http_request(
+ f"https://api.bilibili.com{path}",
+ timeout=timeout,
+ headers={"User-Agent": _UA, "Referer": "https://www.bilibili.com/"},
+ )
+ try:
+ data = json.loads(body)
+ except ValueError:
+ raise ReachError(UPSTREAM_BROKEN, "Bilibili returned a verification page") from None
+ if data.get("code") == -404:
+ raise ReachError(NOT_FOUND, "video not found")
+ if data.get("code") != 0:
+ raise ReachError(UPSTREAM_BROKEN, f"Bilibili API code {data.get('code')}: {data.get('message')}")
+ return data.get("data") or {}
-from .base import Channel
_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
_TIMEOUT = 10
@@ -38,6 +136,41 @@ class BilibiliChannel(Channel):
backends = ["bili-cli", "OpenCLI", "B站搜索 API"]
tier = 1
+ level = "optional"
+ actions = (
+ Action(
+ "search",
+ "Search videos",
+ (
+ Param("query", "what to search for"),
+ Param("limit", "max results", required=False, type=int, default=5),
+ ),
+ (("bili-cli", "_cli_search"),),
+ live_test={"query": "Python", "limit": 2},
+ ),
+ Action(
+ "read",
+ "Video title, uploader, description and stats",
+ (Param("target", "BV id or video link"),),
+ (("bili-cli", "_cli_read"), ("bilibili-api", "_api_read")),
+ live_test={"target": "BV1GJ411x7h7"},
+ ),
+ Action(
+ "hot",
+ "Trending videos",
+ (Param("limit", "max items", required=False, type=int, default=10),),
+ (("bili-cli", "_cli_hot"), ("bilibili-api", "_api_hot")),
+ live_test={"limit": 2},
+ ),
+ Action(
+ "subtitle",
+ "Full subtitle text of a video (needs a Bilibili login)",
+ (Param("target", "BV id or video link"),),
+ (("bili-cli", "_cli_subtitle"),),
+ live_test={"target": "BV1GJ411x7h7"},
+ ),
+ )
+
def can_handle(self, url: str) -> bool:
from agent_reach.utils.url import host_matches
@@ -117,3 +250,34 @@ class BilibiliChannel(Channel):
"B站搜索 API 可达(仅搜索,curl 直连)。"
"完整功能建议安装 bili-cli:pipx install bilibili-cli"
)
+
+ # ── unified entry backends ──
+
+ def _cli_search(self, *, query: str, limit: int, timeout: float) -> list:
+ data = _bili_json(["search", query, "--type", "video", "-n", str(limit)], timeout)
+ return [_cli_video_item(v) for v in (data if isinstance(data, list) else [])[:limit]]
+
+ def _cli_read(self, *, target: str, timeout: float) -> list:
+ data = _bili_json(["video", _bvid(target, timeout)], timeout)
+ return [_cli_video_item(data.get("video") or {})]
+
+ def _api_read(self, *, target: str, timeout: float) -> list:
+ return [_api_video_item(_bili_api(f"/x/web-interface/view?bvid={_bvid(target, timeout)}", timeout))]
+
+ def _cli_hot(self, *, limit: int, timeout: float) -> list:
+ data = _bili_json(["hot", "-n", str(limit)], timeout)
+ return [_cli_video_item(v) for v in (data.get("items") or [])[:limit]]
+
+ def _api_hot(self, *, limit: int, timeout: float) -> list:
+ data = _bili_api(f"/x/web-interface/popular?ps={limit}&pn=1", timeout)
+ return [_api_video_item(v) for v in (data.get("list") or [])[:limit]]
+
+ def _cli_subtitle(self, *, target: str, timeout: float) -> list:
+ data = _bili_json(["video", _bvid(target, timeout), "--subtitle"], timeout)
+ sub = data.get("subtitle") or {}
+ if not sub.get("available"):
+ if not os.path.exists(os.path.expanduser("~/.bilibili-cli/credential.json")):
+ raise ReachError(NEED_LOGIN, "Bilibili is not logged in", hint=_BILI_LOGIN_HINT)
+ raise ReachError(NOT_FOUND, "this video has no subtitles")
+ text = sub.get("text") or "\n".join(i.get("content", "") for i in sub.get("items") or [])
+ return [_cli_video_item(data.get("video") or {}, text=text)]
diff --git a/agent_reach/channels/xueqiu.py b/agent_reach/channels/xueqiu.py
index 1ecf45b5..765837b5 100644
--- a/agent_reach/channels/xueqiu.py
+++ b/agent_reach/channels/xueqiu.py
@@ -4,11 +4,21 @@
import http.cookiejar
import json
import re
+import urllib.error
import urllib.parse
import urllib.request
from typing import Any
-from .base import Channel
+from agent_reach.runtime.result import BAD_INPUT, NEED_LOGIN, NOT_FOUND, ReachError, make_item
+from agent_reach.runtime.run import http_request
+
+from .base import Action, Channel, Param
+
+_XQ_LOGIN_HINT = (
+ "Xueqiu community data needs a browser session. Ask the user to open "
+ "https://xueqiu.com in Chrome, then run `agent-reach configure --from-browser "
+ "chrome --platform xueqiu`. The session expires within hours."
+)
_UA = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
@@ -111,12 +121,115 @@ def _strip_html(text: str) -> str:
return text.strip()
+def _tencent_code(symbol: str) -> str:
+ """SH600519 / 600519 / HK00700 / 00700 / AAPL → Tencent quote code."""
+ s = str(symbol).strip().upper()
+ if re.fullmatch(r"(SH|SZ|BJ)\d{6}", s):
+ return s.lower()
+ if re.fullmatch(r"\d{6}", s):
+ prefix = "sh" if s[0] in "69" else ("bj" if s[0] in "48" else "sz")
+ return prefix + s
+ if re.fullmatch(r"HK\d{1,5}", s):
+ return "hk" + s[2:].zfill(5)
+ if re.fullmatch(r"\d{1,5}", s):
+ return "hk" + s.zfill(5)
+ if re.fullmatch(r"[A-Z][A-Z.\-]{0,9}", s):
+ return "us" + s
+ raise ReachError(BAD_INPUT, "symbol must look like SH600519, 600519, 00700 or AAPL")
+
+
+def _xueqiu_symbol(market: str, code: str) -> str:
+ """Tencent market + code → the symbol Xueqiu uses in URLs and its API."""
+ market = market.lower()
+ if market in ("sh", "sz", "bj"):
+ return market.upper() + code
+ if market == "hk":
+ return code
+ return code.split(".")[0].upper()
+
+
+def _xq_json(url: str) -> Any:
+ try:
+ return _get_json(url)
+ except urllib.error.HTTPError as exc:
+ if exc.code in (400, 401, 403):
+ raise ReachError(NEED_LOGIN, f"Xueqiu rejected the session (HTTP {exc.code})", hint=_XQ_LOGIN_HINT) from None
+ raise
+ except json.JSONDecodeError:
+ raise ReachError(NEED_LOGIN, "Xueqiu returned a verification page instead of data", hint=_XQ_LOGIN_HINT) from None
+
+
+_HOT_POSTS_URL = (
+ "https://xueqiu.com/v4/statuses/public_timeline_by_category.json"
+ "?since_id=-1&max_id=-1&count={limit}&category=-1"
+)
+
+
+def _parse_hot_posts(data: Any, limit: int) -> list:
+ """Each list item carries a JSON-encoded `data` field with the real post."""
+ results = []
+ for item in (data.get("list") or [])[:limit]:
+ try:
+ post = json.loads(item["data"]) if isinstance(item.get("data"), str) else {}
+ except (json.JSONDecodeError, KeyError):
+ post = {}
+ user = post.get("user") or {}
+ text = _strip_html(post.get("text") or post.get("description") or "")
+ target = post.get("target", "")
+ results.append(
+ {
+ "id": post.get("id", 0),
+ "title": post.get("title") or "",
+ "text": text[:200],
+ "author": user.get("screen_name", ""),
+ "likes": post.get("like_count", 0),
+ "url": f"https://xueqiu.com{target}" if target else "",
+ }
+ )
+ return results
+
+
class XueqiuChannel(Channel):
name = "xueqiu"
description = "雪球股票行情与社区动态"
backends = ["Xueqiu API (需要登录 Cookie)"]
tier = 1
+ level = "optional"
+ actions = (
+ Action(
+ "quote",
+ "Real-time stock quote (A-share, HK, US)",
+ (Param("symbol", "e.g. SH600519, 600519, 00700, AAPL"),),
+ (("tencent", "_tencent_quote"), ("xueqiu-api", "_xq_quote")),
+ live_test={"symbol": "SH600519"},
+ ),
+ Action(
+ "search",
+ "Find stocks by name or code",
+ (
+ Param("query", "name or code, e.g. 茅台"),
+ Param("limit", "max results", required=False, type=int, default=5),
+ ),
+ (("tencent", "_tencent_search"), ("xueqiu-api", "_xq_search")),
+ live_test={"query": "茅台", "limit": 3},
+ ),
+ Action(
+ "hot_posts",
+ "Trending Xueqiu community posts (needs a browser session)",
+ (Param("limit", "max items", required=False, type=int, default=10),),
+ (("xueqiu-api", "_xq_hot_posts"),),
+ live_test={"limit": 3},
+ ),
+ Action(
+ "hot_stocks",
+ "Most-watched stocks on Xueqiu (needs a browser session)",
+ (Param("limit", "max items", required=False, type=int, default=10),),
+ (("xueqiu-api", "_xq_hot_stocks"),),
+ live_test={"limit": 3},
+ ),
+ )
+
# ------------------------------------------------------------------ #
# URL routing
# ------------------------------------------------------------------ #
@@ -240,38 +353,7 @@ class XueqiuChannel(Channel):
limit = min(limit, 50)
if limit == 0:
return []
- data = _get_json(
- "https://xueqiu.com/v4/statuses/public_timeline_by_category.json"
- f"?since_id=-1&max_id=-1&count={limit}&category=-1"
- )
- items = data.get("list") or []
- results = []
- for item in items[:limit]:
- # Each item.data is a JSON string containing the real post payload
- try:
- post = (
- json.loads(item["data"])
- if isinstance(item.get("data"), str)
- else {}
- )
- except (json.JSONDecodeError, KeyError):
- post = {}
- user = post.get("user") or {}
- text = _strip_html(
- post.get("text") or post.get("description") or ""
- )
- target = post.get("target", "")
- results.append(
- {
- "id": post.get("id", 0),
- "title": post.get("title") or "",
- "text": text[:200],
- "author": user.get("screen_name", ""),
- "likes": post.get("like_count", 0),
- "url": f"https://xueqiu.com{target}" if target else "",
- }
- )
- return results
+ return _parse_hot_posts(_get_json(_HOT_POSTS_URL.format(limit=limit)), limit)
def get_hot_stocks(self, limit: int = 10, stock_type: int = 10) -> list:
"""获取热门股票排行。
@@ -300,3 +382,91 @@ class XueqiuChannel(Channel):
}
)
return results
+
+ # ── unified entry backends ──
+
+ def _tencent_quote(self, *, symbol: str, timeout: float) -> list:
+ code = _tencent_code(symbol)
+ body = http_request(f"https://qt.gtimg.cn/q={code}", timeout=timeout).decode("gbk", errors="replace")
+ m = re.search(r'="([^"]*)"', body)
+ if not m or m.group(1).count("~") < 34:
+ raise ReachError(NOT_FOUND, f"no quote for {symbol}")
+ f = m.group(1).split("~")
+ xq = _xueqiu_symbol(code[:2], f[2])
+ quote = {
+ "symbol": xq, "name": f[1], "current": f[3], "last_close": f[4], "open": f[5],
+ "chg": f[31], "percent": f[32], "high": f[33], "low": f[34], "time": f[30],
+ }
+ return [
+ make_item(
+ title=f"{f[1]} ({xq})",
+ url=f"https://xueqiu.com/S/{xq}",
+ text=f"{f[1]} {f[3]} ({f[32]}%)",
+ raw=quote,
+ )
+ ]
+
+ def _tencent_search(self, *, query: str, limit: int, timeout: float) -> list:
+ q = urllib.parse.quote(query)
+ body = http_request(f"https://smartbox.gtimg.cn/s3/?v=2&q={q}&t=all", timeout=timeout).decode("utf-8", errors="replace")
+ m = re.search(r'="([^"]*)"', body)
+ if not m or m.group(1) in ("", "N"):
+ return []
+ items = []
+ for entry in json.loads(f'"{m.group(1)}"').split("^")[:limit]:
+ parts = entry.split("~")
+ if len(parts) < 3:
+ continue
+ market, code, name = parts[0], parts[1], parts[2]
+ xq = _xueqiu_symbol(market, code)
+ items.append(
+ make_item(
+ title=f"{name} ({xq})",
+ url=f"https://xueqiu.com/S/{xq}",
+ raw={"symbol": xq, "name": name, "market": market, "type": parts[4] if len(parts) > 4 else None},
+ )
+ )
+ return items
+
+ def _xq_quote(self, *, symbol: str, timeout: float) -> list:
+ code = _tencent_code(symbol)
+ xq = _xueqiu_symbol(code[:2], code[2:])
+ data = _xq_json(f"https://stock.xueqiu.com/v5/stock/quote.json?symbol={urllib.parse.quote(xq)}&extend=detail")
+ q = (data.get("data") or {}).get("quote")
+ if not q:
+ raise ReachError(NOT_FOUND, f"no quote for {symbol}")
+ return [
+ make_item(
+ title=f"{q.get('name')} ({q.get('symbol')})",
+ url=f"https://xueqiu.com/S/{q.get('symbol')}",
+ text=f"{q.get('name')} {q.get('current')} ({q.get('percent')}%)",
+ raw=q,
+ )
+ ]
+
+ def _xq_search(self, *, query: str, limit: int, timeout: float) -> list:
+ data = _xq_json(f"https://xueqiu.com/stock/search.json?code={urllib.parse.quote(query)}&size={limit}")
+ return [
+ make_item(title=f"{s.get('name')} ({s.get('code')})", url=f"https://xueqiu.com/S/{s.get('code')}", raw=s)
+ for s in (data.get("stocks") or [])[:limit]
+ ]
+
+ def _xq_hot_posts(self, *, limit: int, timeout: float) -> list:
+ limit = max(0, min(limit, 50))
+ posts = _parse_hot_posts(_xq_json(_HOT_POSTS_URL.format(limit=limit)), limit) if limit else []
+ return [
+ make_item(title=p["title"] or p["text"][:60], url=p["url"] or None, author=p["author"], text=p["text"], raw=p)
+ for p in posts
+ ]
+
+ def _xq_hot_stocks(self, *, limit: int, timeout: float) -> list:
+ data = _xq_json(f"https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size={limit}&type=10")
+ return [
+ make_item(
+ title=f"{s.get('name')} ({s.get('code') or s.get('symbol')})",
+ url=f"https://xueqiu.com/S/{s.get('code') or s.get('symbol')}",
+ text=f"{s.get('name')} {s.get('current')} ({s.get('percent')}%)",
+ raw=s,
+ )
+ for s in ((data.get("data") or {}).get("items") or [])[:limit]
+ ]