Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.
Python agent-infrastructureai-agentai-searchautomationbilibiliclaude-codeclicursorfree-apillm-toolsmcppythonreddit-scrapertwitter-scraperweb-scraperxiaohongshuyoutube-transcript
feat: complete channel menu install flow + XiaoHongShu channel
Panniantong1b8b202e1ed0
diff --git a/agent_eyes/channels/__init__.py b/agent_eyes/channels/__init__.pyindex 999eb684..37e964e7 100644--- a/agent_eyes/channels/__init__.py+++ b/agent_eyes/channels/__init__.py@@ -19,6 +19,7 @@ from .reddit import RedditChannelfrom .rss import RSSChannelfrom .bilibili import BilibiliChannelfrom .exa_search import ExaSearchChannel+from .xiaohongshu import XiaoHongShuChannel# Channel registry — order matters (first match wins, web is last as fallback)@@ -28,6 +29,7 @@ ALL_CHANNELS: List[Channel] = [YouTubeChannel(),RedditChannel(),BilibiliChannel(),+ XiaoHongShuChannel(),RSSChannel(),ExaSearchChannel(),WebChannel(), # Fallback — handles any URLdiff --git a/agent_eyes/channels/xiaohongshu.py b/agent_eyes/channels/xiaohongshu.pynew file mode 100644index 00000000..e0e0a752--- /dev/null+++ b/agent_eyes/channels/xiaohongshu.py@@ -0,0 +1,110 @@+# -*- coding: utf-8 -*-+"""XiaoHongShu (小红书) — via cookie-based API access.++Backend: XHS web API + cookies+Swap to: any XHS access method+"""++import re+import json+import requests+from urllib.parse import urlparse+from .base import Channel, ReadResult+++class XiaoHongShuChannel(Channel):+ name = "xiaohongshu"+ description = "XiaoHongShu (小红书) notes"+ backends = ["XHS Web API"]+ requires_config = ["xhs_cookie"]+ tier = 2++ def can_handle(self, url: str) -> bool:+ domain = urlparse(url).netloc.lower()+ return "xiaohongshu.com" in domain or "xhslink.com" in domain++ async def read(self, url: str, config=None) -> ReadResult:+ cookie = config.get("xhs_cookie") if config else None++ if not cookie:+ # Fallback to Jina Reader (works for some public notes)+ from agent_eyes.channels.web import WebChannel+ return await WebChannel().read(url, config)++ # Extract note ID from URL+ note_id = self._extract_note_id(url)+ if not note_id:+ from agent_eyes.channels.web import WebChannel+ return await WebChannel().read(url, config)++ headers = {+ "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",+ "Cookie": cookie,+ "Referer": "https://www.xiaohongshu.com/",+ }++ # Fetch note page+ resp = requests.get(+ f"https://www.xiaohongshu.com/explore/{note_id}",+ headers=headers,+ timeout=15,+ )+ resp.raise_for_status()+ html = resp.text++ # Extract note data from HTML+ title, content, author = self._parse_html(html)++ return ReadResult(+ title=title or f"XHS Note {note_id}",+ content=content or "Could not extract content. Cookie may be expired.",+ url=url,+ author=author,+ platform="xiaohongshu",+ )++ def _extract_note_id(self, url: str) -> str:+ """Extract note ID from various XHS URL formats."""+ # https://www.xiaohongshu.com/explore/xxxxx+ # https://www.xiaohongshu.com/discovery/item/xxxxx+ # https://xhslink.com/xxxxx+ path = urlparse(url).path+ parts = path.strip("/").split("/")+ if parts:+ return parts[-1]+ return ""++ def _parse_html(self, html: str):+ """Extract title, content, author from XHS HTML."""+ title = ""+ content = ""+ author = ""++ # Try to find JSON data in page+ match = re.search(r'window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>', html, re.DOTALL)+ if match:+ try:+ # XHS embeds note data in initial state+ state = json.loads(match.group(1).replace('undefined', 'null'))+ note_data = state.get("note", {}).get("noteDetailMap", {})+ if note_data:+ first_note = list(note_data.values())[0]+ note = first_note.get("note", {})+ title = note.get("title", "")+ content = note.get("desc", "")+ author = note.get("user", {}).get("nickname", "")+ except (json.JSONDecodeError, KeyError, IndexError):+ pass++ # Fallback: extract from meta tags+ if not title:+ m = re.search(r'<title>(.*?)</title>', html)+ if m:+ title = m.group(1)++ if not content:+ m = re.search(r'<meta name="description" content="(.*?)"', html)+ if m:+ content = m.group(1)++ return title, content, authordiff --git a/agent_eyes/cli.py b/agent_eyes/cli.pyindex 50e05a4b..f87f4541 100644--- a/agent_eyes/cli.py+++ b/agent_eyes/cli.py@@ -81,9 +81,10 @@ def main():# ── configure ──p_conf = sub.add_parser("configure", help="Set a config value")- p_conf.add_argument("key", choices=["exa-key", "proxy", "github-token", "groq-key"],+ p_conf.add_argument("key", choices=["exa-key", "proxy", "github-token", "groq-key",+ "twitter-cookies", "xhs-cookie", "youtube-cookies"],help="What to configure")- p_conf.add_argument("value", help="The value to set")+ p_conf.add_argument("value", nargs="+", help="The value(s) to set")# ── doctor ──sub.add_parser("doctor", help="Check platform availability")@@ -178,28 +179,36 @@ def _cmd_install(args):def _cmd_configure(args):"""Set a config value and test it."""from agent_eyes.config import Config- import subprocessconfig = Config()+ value = " ".join(args.value) if isinstance(args.value, list) else args.value- key_map = {- "exa-key": "exa_api_key",- "proxy": ("reddit_proxy", "bilibili_proxy"),- "github-token": "github_token",- "groq-key": "groq_api_key",- }-- config_key = key_map.get(args.key)- if isinstance(config_key, tuple):- for k in config_key:- config.set(k, args.value)- else:- config.set(config_key, args.value)+ if args.key == "proxy":+ config.set("reddit_proxy", value)+ config.set("bilibili_proxy", value)+ print(f"✅ Proxy configured for Reddit + Bilibili!")++ # Auto-test+ print("Testing Reddit access...", end=" ")+ try:+ import requests+ resp = requests.get(+ "https://www.reddit.com/r/test.json?limit=1",+ headers={"User-Agent": "Mozilla/5.0"},+ proxies={"http": value, "https": value},+ timeout=10,+ )+ if resp.status_code == 200:+ print("✅ Reddit works!")+ else:+ print(f"⚠️ Reddit returned {resp.status_code}")+ except Exception as e:+ print(f"❌ Failed: {e}")- print(f"✅ {args.key} configured!")+ elif args.key == "exa-key":+ config.set("exa_api_key", value)+ print(f"✅ Exa key configured!")- # Auto-test- if args.key == "exa-key":print("Testing search...", end=" ")try:import asyncio@@ -209,27 +218,73 @@ def _cmd_configure(args):if results:print("✅ Search works!")else:- print("⚠️ No results, but API connected.")+ print("⚠️ No results, but API connected.")except Exception as e:print(f"❌ Failed: {e}")- elif args.key == "proxy":- print("Testing Reddit access...", end=" ")+ elif args.key == "twitter-cookies":+ # Expect: auth_token ct0+ parts = value.split()+ if len(parts) == 2:+ config.set("twitter_auth_token", parts[0])+ config.set("twitter_ct0", parts[1])+ print(f"✅ Twitter cookies configured!")++ print("Testing Twitter access...", end=" ")+ try:+ import subprocess+ result = subprocess.run(+ ["birdx", "search", "test", "-n", "1",+ "--auth-token", parts[0], "--ct0", parts[1]],+ capture_output=True, text=True, timeout=15,+ )+ if result.returncode == 0 and result.stdout.strip():+ print("✅ Twitter Advanced works!")+ else:+ print(f"⚠️ Test returned no results (cookies might be wrong)")+ except FileNotFoundError:+ print("⚠️ birdx not installed. Run: pip install birdx")+ except Exception as e:+ print(f"❌ Failed: {e}")+ else:+ print("❌ Usage: agent-eyes configure twitter-cookies AUTH_TOKEN CT0")+ print(" (two values separated by space)")++ elif args.key == "xhs-cookie":+ config.set("xhs_cookie", value)+ print(f"✅ XiaoHongShu cookie configured!")++ print("Testing XHS access...", end=" ")try:import requestsresp = requests.get(- "https://www.reddit.com/r/test.json?limit=1",- headers={"User-Agent": "Mozilla/5.0"},- proxies={"http": args.value, "https": args.value},+ "https://www.xiaohongshu.com/",+ headers={+ "User-Agent": "Mozilla/5.0",+ "Cookie": value,+ },timeout=10,)- if resp.status_code == 200:- print("✅ Reddit accessible!")+ if resp.status_code == 200 and "xiaohongshu" in resp.text.lower():+ print("✅ XiaoHongShu works!")else:- print(f"❌ Reddit returned {resp.status_code}")+ print(f"⚠️ Got status {resp.status_code}, cookie might be expired")except Exception as e:print(f"❌ Failed: {e}")+ elif args.key == "youtube-cookies":+ config.set("youtube_cookies_from", value)+ print(f"✅ YouTube cookie source configured: {value}")+ print(" yt-dlp will use cookies from this browser for age-restricted/member videos.")++ elif args.key == "github-token":+ config.set("github_token", value)+ print(f"✅ GitHub token configured!")++ elif args.key == "groq-key":+ config.set("groq_api_key", value)+ print(f"✅ Groq key configured!")+def _cmd_doctor():from agent_eyes.config import Configdiff --git a/docs/install.md b/docs/install.mdindex 7cc87c16..19d39edf 100644--- a/docs/install.md+++ b/docs/install.md@@ -16,7 +16,7 @@ Your Agent will handle everything. You just answer a few questions.## For AI AgentsYou are installing Agent Eyes — giving yourself eyes to see the entire internet.-Follow this guide step by step. Do not skip steps.+Follow this guide step by step. Do not improvise.### Step 1: Install@@ -31,148 +31,197 @@ Verify:agent-eyes version```-### Step 2: Ask environment+### Step 2: Detect environmentAsk the user:-> "You're installing Agent Eyes! First question: are you running on a **local computer** or a **server/VPS**?"+> "First question: are you running on a **local computer** or a **server/VPS**?"-Remember their answer — it affects which channels need extra setup.+This determines what works out of the box and what needs extra setup.### Step 3: Show channel menu-Show the user this menu. Explain each option clearly.+Present this menu to the user. Adjust the details based on their environment (local vs server).-**Present it like this** (adjust the Bilibili line based on Step 2 answer):+**IMPORTANT**: Make sure the user understands what each tier means and what they get/miss.----> Here are all the channels Agent Eyes supports. I'll help you set up whichever ones you want.+> **Agent Eyes supports these channels. Let me help you choose what to set up.**>-> **Already active — no setup needed ✅**-> These work right now, nothing to configure:-> - ✅ **Web pages** — read any URL-> - ✅ **GitHub** — read repos, issues, PRs + search repos-> - ✅ **RSS** — read any RSS/Atom feed-> - ✅ **YouTube** — video transcripts via yt-dlp-> - ✅ **Twitter basic** — read individual tweets+> --->-> **Recommended — free, takes 30 seconds each 🔍**-> - 🔍 **Exa Search** (STRONGLY RECOMMENDED)-> - What it unlocks: **full web search + Reddit search + Twitter search**-> - What you need: one free API key from https://exa.ai (1000 searches/month free)-> - Without it: I can read URLs you give me, but can't search the internet for you+> **🟢 Already active — works right now, nothing needed:**>-> **Optional — easy setup 🔧**-> - 🐦 **Twitter Advanced** — search timelines, read threads, deep search-> - What you need: export cookies from your browser (free, 2 minutes)-> - Without it: I can still search Twitter via Exa and read individual tweets+> | Channel | What it does |+> |---------|-------------|+> | ✅ Web pages | Read any URL on the internet |+> | ✅ GitHub | Read repos, issues, PRs + search repositories |+> | ✅ RSS | Read any RSS/Atom feed |>-> - 📺 **Bilibili** [IF LOCAL: "already works ✅"] [IF SERVER: "⚠️ needs proxy — Bilibili blocks server IPs. Proxy costs ~$1/month at webshare.io. Without it, Bilibili won't work from your server."]+> These 3 channels are always available. No setup needed.>-> **Optional — more setup 🔨**-> - 📖 **Reddit Full Reader** — read complete posts + all comments-> - What you need: residential proxy (~$1/month at https://webshare.io)-> - Without it: I can still **search** Reddit content via Exa (free!), just can't read full posts with all comments-> - [IF SERVER: "Same proxy works for both Reddit and Bilibili"]+> --->-> - 💬 **WeChat** — read WeChat public articles-> - What you need: install browser component (~150MB download)+> **🔵 Cookie-based — free, just need to export cookies from your browser (~2 min each):**>-> - 📕 **XiaoHongShu** — read XHS notes-> - What you need: browser component + scan QR code once to login+> | Channel | What it does | Without cookies |+> |---------|-------------|-----------------|+> | 🐦 Twitter/X | Read tweets, search timelines, read threads | Can still read individual tweet URLs (basic mode) |+> | 📺 YouTube | Video transcripts (public + age-restricted) | Public videos work without cookies; age-restricted/member-only videos need cookies |+> | 📺 Bilibili | Video info + subtitles | [LOCAL: works without cookies ✅] [SERVER: ⚠️ blocked — needs proxy, see below] |+> | 📕 XiaoHongShu | Read XHS notes + comments | Falls back to basic web reading (less content) |>-> **Which ones would you like me to set up?** (I recommend at least Exa Search)+> **How to export cookies**: open the site in Chrome → F12 → Application → Cookies → copy the values. I'll walk you through it for each one you choose.+>+> ---+>+> **🟡 Free API key — one key unlocks 3 search capabilities (~30 seconds):**+>+> | Channel | What it unlocks |+> |---------|----------------|+> | 🔍 Exa Search (RECOMMENDED) | **Web search + Reddit search + Twitter search** — one free key powers all three |+>+> Without Exa: I can read URLs you give me, but I cannot search the internet for you.+> With Exa: I can find information across the entire web, Reddit, and Twitter.+>+> Free tier: 1000 searches/month at https://exa.ai — more than enough for most users.+>+> ---+>+> **🟠 Proxy-based — needed only on servers, $1/month shared across channels:**+>+> | Channel | Why proxy needed | Without proxy |+> |---------|-----------------|---------------|+> | 📖 Reddit | Reddit blocks server IPs | **Reddit search still works via Exa (free!)** — you just can't read full posts with all comments |+> | 📺 Bilibili | Bilibili blocks server IPs | Won't work from server |+>+> [ONLY SHOW THIS SECTION IF USER SAID "SERVER"]+> [IF LOCAL: Skip this section — Reddit and Bilibili work directly on local computers]+>+> One proxy works for both. Recommend: https://www.webshare.io ($1/month residential proxy).+> Same proxy also works for Bilibili cookies-based access.+>+> ---+>+> **Which channels do you want me to set up?**+> (I recommend: **Exa Search** for sure, plus any cookie-based channels you actually use)----Wait for user to choose. Then proceed to Step 4.+Wait for the user to choose. Then go to Step 4.-### Step 4: Run installer+### Step 4: Run base installer```bashagent-eyes install --env=<local|server>```-### Step 5: Configure selected channels+### Step 5: Configure each selected channel++Run the corresponding command for each channel the user chose.+Every configure command auto-tests and gives instant feedback (✅ or ❌).-For each channel the user selected, run the corresponding configure command.-Each command auto-tests after configuring.+----#### If user selected: Exa Search+#### Exa Search-Tell the user:-> "Go to https://exa.ai, sign up (free), and send me your API Key."+Tell user: "Go to https://exa.ai, sign up (free), copy your API Key, and send it to me."-When they provide it:```bashagent-eyes configure exa-key THE_KEY```-Expected: `✅ exa-key configured! Testing search... ✅ Search works!`-#### If user selected: Reddit Full Reader and/or Bilibili (on server)+----Tell the user:-> "Reddit and Bilibili need a residential proxy. Steps:-> 1. Go to https://www.webshare.io-> 2. Sign up and get a proxy ($1/month plan is enough)-> 3. Copy your proxy address (format: http://username:password@ip:port)-> 4. Send it to me"+#### Twitter/X++Tell user:+> "Open https://x.com in Chrome (make sure you're logged in).+> Press F12 → Application → Cookies → https://x.com+> Find these two values: **auth_token** and **ct0**+> Copy both and send them to me."-When they provide it:```bash-agent-eyes configure proxy THE_PROXY_URL+agent-eyes configure twitter-cookies AUTH_TOKEN CT0```-Expected: `✅ proxy configured! Testing Reddit... ✅ Reddit accessible!`-#### If user selected: Twitter Advanced+---++#### YouTube (cookies for age-restricted videos)-Tell the user:-> "Twitter Advanced needs cookies from your browser. Steps:-> 1. Open https://x.com in Chrome and make sure you're logged in-> 2. Press F12 → Application tab → Cookies → https://x.com-> 3. Find and copy these two values: **auth_token** and **ct0**-> 4. Send them to me"+Tell user:+> "Most YouTube videos work without any setup. If you want access to age-restricted or member-only videos:+> Which browser do you use? (chrome/firefox/edge/safari)"-When they provide cookies:```bash-# birdx cookie setup (if birdx is available)-which birdx || pip install birdx-# Configure birdx with cookies (method varies by birdx version)+agent-eyes configure youtube-cookies chrome```+(replace `chrome` with whatever browser they use — yt-dlp reads cookies directly from the browser)-#### If user selected: WeChat+---++#### Bilibili++**If local**: Already works. No setup needed.++**If server**: Needs proxy (same proxy as Reddit, see Proxy section below).++If user also wants cookies for member-only content:+> "Open https://bilibili.com in Chrome (logged in).+> F12 → Application → Cookies → find **SESSDATA**+> Send it to me."++(For now, Bilibili public API works without cookies. Cookie support is for future member-only content.)++---++#### XiaoHongShu++Tell user:+> "Open https://www.xiaohongshu.com in Chrome (make sure you're logged in).+> Press F12 → Application → Cookies → https://www.xiaohongshu.com+> Select all cookies, right-click → Copy All+> Or just copy the full cookie string from a network request header.+> Send it to me."```bash-pip install agent-eyes[browser]-playwright install chromium+agent-eyes configure xhs-cookie THE_COOKIE_STRING```-Tell user: "✅ WeChat is ready! Send me any WeChat article URL."-#### If user selected: XiaoHongShu+---++#### Proxy (Reddit + Bilibili on server)++Tell user:+> "Reddit and Bilibili block server IPs. You need a residential proxy to access them.+> 1. Go to https://www.webshare.io+> 2. Sign up and get a residential proxy ($1/month is enough)+> 3. Copy your proxy address (format: http://username:password@ip:port)+> 4. Send it to me"```bash-pip install agent-eyes[browser]-playwright install chromium+agent-eyes configure proxy THE_PROXY_URL```-Tell user: "I'll need you to scan a QR code once to log in. Ready?" Then read the guide: read the file at `agent_eyes/guides/setup-xiaohongshu.md` for detailed steps.-### Step 6: Final check+This configures the proxy for both Reddit and Bilibili at once.++---++### Step 6: Final health check```bashagent-eyes doctor```-Summarize results for the user naturally. Example:+Summarize what's working for the user. Example:> "✅ All done! Here's what I can do now:-> - 📖 Read: Web, GitHub, YouTube, RSS, Tweets, Reddit, Bilibili-> - 🔍 Search: Web, GitHub, Reddit, Twitter>-> Try asking me things like:-> - 'Search Reddit for best self-hosted LLM'-> - 'Read this GitHub repo: [URL]'-> - 'What's trending on Twitter about AI agents?'"+> 📖 **Read**: Web, GitHub, YouTube, Bilibili, Twitter, XiaoHongShu, RSS+> 🔍 **Search**: Web, GitHub, Reddit, Twitter+>+> Try it out — ask me to search or read anything!"---@@ -181,11 +230,14 @@ Summarize results for the user naturally. Example:| Action | Command ||--------|---------|| Install | `pip install git+https://github.com/Panniantong/agent-eyes.git` |-| Setup | `agent-eyes install --env=<local\|server>` |+| Base setup | `agent-eyes install --env=<local\|server>` || Add Exa key | `agent-eyes configure exa-key YOUR_KEY` |+| Add Twitter | `agent-eyes configure twitter-cookies AUTH_TOKEN CT0` |+| Add YouTube | `agent-eyes configure youtube-cookies chrome` |+| Add XiaoHongShu | `agent-eyes configure xhs-cookie COOKIE_STRING` || Add proxy | `agent-eyes configure proxy http://user:pass@ip:port` || Health check | `agent-eyes doctor` |-| Read any URL | `agent-eyes read <url>` |+| Read URL | `agent-eyes read <url>` || Search web | `agent-eyes search "query"` || Search GitHub | `agent-eyes search-github "query"` || Search Reddit | `agent-eyes search-reddit "query"` |