Skip to content

Give your AI agent eyes to see the entire internet. Read & search Twitter, Reddit, YouTube, GitHub, Bilibili, XiaoHongShu — one CLI, zero API fees.

8.7 KB
Raw Blame
# -*- coding: utf-8 -*-
"""Web — search (Exa) and read any page (Jina Reader, Exa fallback).

This channel is a fallback: agents that already have working built-in web
search / fetch should use those first.
"""

import http.client
import json
import re
import time
import urllib.request

from agent_reach.runtime.result import (
    BAD_INPUT,
    RATE_LIMITED,
    UPSTREAM_BROKEN,
    ReachError,
    make_item,
)
from agent_reach.runtime.run import http_request
from agent_reach.utils.url import normalize_public_http_url

from .base import Action, Channel, Param

_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
_MAX_RESPONSE_BYTES = 5 * 1024 * 1024
_ANTIBOT_SCAN_BYTES = 4096
_EXA_MCP = "https://mcp.exa.ai/mcp"
_EXA_HEADER_RE = re.compile(r"^(Title|URL|Published|Author):\s*(.*)$")
_EXA_LIMIT_RE = re.compile(
    r"\b429\b|rate[ _-]?limit|too many requests|\bquota\b|"
    r"(?:usage|request|daily|monthly|free)[ _-]+limit|"
    r"(?:credit|credits|balance).{0,40}(?:exhaust|deplet|insufficient|exceed|reach)|"
    r"(?:out of|insufficient|exhausted).{0,20}credits?",
    re.IGNORECASE,
)


def _exa_error(text: str, rpc_code=None) -> ReachError:
    if rpc_code in (-32602, "-32602"):
        return ReachError(BAD_INPUT, f"Exa rejected the arguments: {text[:300]}")
    if rpc_code in (429, "429") or _EXA_LIMIT_RE.search(text):
        return ReachError(
            RATE_LIMITED, f"Exa: {text[:300]}",
            hint="Exa's hosted search limit was reached. Use the agent's built-in web search or wait before retrying.",
        )
    return ReachError(UPSTREAM_BROKEN, f"Exa: {text[:300]}")


def _is_antibot_page(body: bytes) -> bool:
    """Recognize high-confidence Jina/Cloudflare challenge responses."""
    sample = body[:_ANTIBOT_SCAN_BYTES].decode("utf-8", errors="ignore").casefold()

    jina_captcha_warning = "warning:" in sample and "requiring captcha" in sample
    challenge_structure = any(
        marker in sample
        for marker in (
            "title: just a moment...",
            "## performing security verification",
            "title: attention required! | cloudflare",
        )
    )
    cloudflare_block = "title: attention required! | cloudflare" in sample and (
        "ray id" in sample or "/cdn-cgi/challenge-platform/" in sample
    )
    return (jina_captcha_warning and challenge_structure) or cloudflare_block


def _public_url(target: str) -> str:
    try:
        return normalize_public_http_url(target)
    except ValueError:
        raise ReachError(BAD_INPUT, "target must be a public http(s) URL") from None


def _exa_call(tool: str, arguments: dict, timeout: float) -> str:
    """Call one tool on Exa's free hosted MCP (no key) and return its text."""
    body = json.dumps(
        {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": arguments}}
    ).encode("utf-8")
    started = time.monotonic()
    for attempt in range(2):
        try:
            raw = http_request(
                _EXA_MCP, timeout=max(0.1, timeout - (time.monotonic() - started)), data=body,
                headers={"Content-Type": "application/json", "Accept": "application/json, text/event-stream"},
            ).decode("utf-8", errors="replace")
            break
        except http.client.IncompleteRead:
            # Read-only provider requests: one retry for a truncated transport,
            # within the original budget. Never retry HTTP 429/403 or tool errors.
            if attempt or time.monotonic() - started >= timeout:
                raise
    payloads = [ln[5:].strip() for ln in raw.splitlines() if ln.startswith("data:")] or [raw]
    msg = json.loads(payloads[-1])
    if "error" in msg:
        text = str(msg["error"].get("message", msg["error"]))
        raise _exa_error(text, msg["error"].get("code"))
    result = msg.get("result") or {}
    text = "\n".join(c.get("text", "") for c in result.get("content", []) if c.get("type") == "text")
    if result.get("isError"):
        raise _exa_error(text)
    return text


def _parse_exa_text(text: str) -> list:
    """Split Exa's `Title: / URL: / Published: / Author:` blocks into items."""
    items = []
    for block in re.split(r"\n(?=Title: )", "\n" + text.strip()):
        block = block.strip()
        if not block.startswith("Title:"):
            continue
        fields, body = {}, []
        lines = block.splitlines()
        i = 0
        while i < len(lines):
            m = _EXA_HEADER_RE.match(lines[i])
            if not m:
                break
            fields[m.group(1).lower()] = m.group(2).strip()
            i += 1
        for line in lines[i:]:
            if line.strip() in ("Highlights:", "Text:"):
                continue
            body.append(line)
        na = lambda v: None if v in (None, "", "N/A") else v  # noqa: E731
        items.append(
            make_item(
                title=na(fields.get("title")),
                url=na(fields.get("url")),
                author=na(fields.get("author")),
                published_at=na(fields.get("published")),
                text="\n".join(body).strip(),
                raw=fields,
            )
        )
    return items


def _parse_jina(url: str, body: str) -> dict:
    title = re.search(r"^Title:\s*(.*)$", body, re.M)
    published = re.search(r"^Published Time:\s*(.*)$", body, re.M)
    content = body.split("Markdown Content:", 1)[1] if "Markdown Content:" in body else body
    return make_item(
        title=title.group(1).strip() if title else None,
        url=url,
        published_at=published.group(1).strip() if published else None,
        text=content.strip(),
    )


class WebChannel(Channel):
    name = "web"
    description = "网页搜索与阅读(Agent 自带工具不可用时的保底)"

    level = "default"
    actions = (
        Action(
            "search",
            "Search the web (Exa semantic search)",
            (
                Param("query", "what to search for"),
                Param("limit", "max results", required=False, type=int, default=5),
            ),
            (("exa", "_exa_search"),),
            live_test={"query": "yt-dlp github repository", "limit": 2},
        ),
        Action(
            "read",
            "Read a web page as clean text",
            (Param("target", "page URL"),),
            (("jina", "_jina_read"), ("exa", "_exa_read")),
            live_test={"target": "https://example.com"},
        ),
    )

    def can_handle(self, url: str) -> bool:
        return True  # Fallback — handles any URL

    def read(self, url: str, timeout: float = 30) -> str:
        """通过 Jina Reader 读取网页,返回 Markdown 全文。"""
        url = normalize_public_http_url(url)
        jina_url = f"https://r.jina.ai/{url}"
        req = urllib.request.Request(
            jina_url,
            headers={"User-Agent": _UA, "Accept": "text/plain"},
        )
        with urllib.request.urlopen(req, timeout=timeout) as resp:
            body = resp.read(_MAX_RESPONSE_BYTES + 1)
        if len(body) > _MAX_RESPONSE_BYTES:
            raise ValueError(
                f"Jina Reader response exceeds {_MAX_RESPONSE_BYTES} byte limit"
            )
        if _is_antibot_page(body):
            raise RuntimeError(
                "Jina Reader 返回了反爬验证页,未获取到目标内容;"
                "请改用站点专用工具或浏览器读取"
            )
        return body.decode("utf-8")

    # ── unified entry backends ──

    def _exa_search(self, *, query: str, limit: int, timeout: float) -> list:
        text = _exa_call("web_search_exa", {"query": query, "objective": query, "numResults": limit}, timeout)
        items = _parse_exa_text(text)
        if not items:
            # Do not report an unrecognized response as a successful zero-hit
            # search. The hosted MCP does not give us a verified empty schema.
            raise ReachError(UPSTREAM_BROKEN, "Exa returned no recognizable search results")
        return items[:limit]

    def _exa_read(self, *, target: str, timeout: float) -> list:
        url = _public_url(target)
        text = _exa_call("web_fetch_exa", {"urls": [url], "maxCharacters": 20000}, timeout)
        items = _parse_exa_text(text)
        if items:
            items[0]["url"] = items[0]["url"] or url
            return items[:1]
        if not text.strip():
            raise ReachError(UPSTREAM_BROKEN, "Exa returned no content")
        return [make_item(url=url, text=text.strip())]

    def _jina_read(self, *, target: str, timeout: float) -> list:
        url = _public_url(target)
        try:
            body = self.read(url, timeout=timeout)
        except RuntimeError as exc:
            raise ReachError(UPSTREAM_BROKEN, str(exc)) from None
        return [_parse_jina(url, body)]