Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Web tools: web_fetch."""
import html
import json
import re
from html.parser import HTMLParser
from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
import httpx
from vikingbot.agent.tools.base import Tool
if TYPE_CHECKING:
from vikingbot.agent.tools.base import ToolContext
# Shared constants
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36"
MAX_REDIRECTS = 5 # Limit redirects to prevent DoS attacks
class _HTMLTextExtractor(HTMLParser):
"""Extract text content while ignoring script/style blocks."""
def __init__(self):
super().__init__(convert_charrefs=True)
self._parts: list[str] = []
self._ignored_tag_stack = 0
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]):
if tag.lower() in {"script", "style"}:
self._ignored_tag_stack += 1
def handle_endtag(self, tag: str):
if tag.lower() in {"script", "style"} and self._ignored_tag_stack:
self._ignored_tag_stack -= 1
def handle_data(self, data: str):
if not self._ignored_tag_stack:
self._parts.append(data)
def get_text(self) -> str:
return "".join(self._parts)
def _strip_tags(text: str) -> str:
"""Remove HTML tags and decode entities."""
parser = _HTMLTextExtractor()
parser.feed(text)
parser.close()
return html.unescape(parser.get_text()).strip()
def _normalize(text: str) -> str:
"""Normalize whitespace."""
text = re.sub(r"[ \t]+", " ", text)
return re.sub(r"\n{3,}", "\n\n", text).strip()
def _validate_url(url: str) -> tuple[bool, str]:
"""Validate URL: must be http(s) with valid domain."""
try:
p = urlparse(url)
if p.scheme not in ("http", "https"):
return False, f"Only http/https allowed, got '{p.scheme or 'none'}'"
if not p.netloc:
return False, "Missing domain"
return True, ""
except Exception as e:
return False, str(e)
class WebFetchTool(Tool):
"""Fetch and extract content from a URL using Readability."""
name = "web_fetch"
description = "Fetch URL and extract readable content (HTML → markdown/text)."
parameters = {
"type": "object",
"properties": {
"url": {"type": "string", "description": "URL to fetch"},
"extractMode": {"type": "string", "enum": ["markdown", "text"], "default": "markdown"},
"maxChars": {"type": "integer", "minimum": 100},
},
"required": ["url"],
}
def __init__(self, max_chars: int = 50000):
self.max_chars = max_chars
async def execute(
self,
tool_context: "ToolContext",
url: str,
extractMode: str = "markdown",
maxChars: int | None = None,
**kwargs: Any,
) -> str:
from readability import Document
max_chars = maxChars or self.max_chars
# Validate URL before fetching
is_valid, error_msg = _validate_url(url)
if not is_valid:
return "Error: " + json.dumps(
{"error": f"URL validation failed: {error_msg}", "url": url}
)
try:
async with httpx.AsyncClient(
follow_redirects=True, max_redirects=MAX_REDIRECTS, timeout=30.0
) as client:
r = await client.get(url, headers={"User-Agent": USER_AGENT})
r.raise_for_status()
ctype = r.headers.get("content-type", "")
# JSON
if "application/json" in ctype:
text, extractor = json.dumps(r.json(), indent=2), "json"
# HTML
elif "text/html" in ctype or r.text[:256].lower().startswith(("<!doctype", "<html")):
doc = Document(r.text)
content = (
self._to_markdown(doc.summary())
if extractMode == "markdown"
else _strip_tags(doc.summary())
)
text = f"# {doc.title()}\n\n{content}" if doc.title() else content
extractor = "readability"
else:
text, extractor = r.text, "raw"
truncated = len(text) > max_chars
if truncated:
text = text[:max_chars]
return json.dumps(
{
"url": url,
"finalUrl": str(r.url),
"status": r.status_code,
"extractor": extractor,
"truncated": truncated,
"length": len(text),
"text": text,
}
)
except Exception as e:
return "Error: " + json.dumps({"error": str(e), "url": url})
def _to_markdown(self, html: str) -> str:
"""Convert HTML to markdown."""
# Convert links, headings, lists before stripping tags
text = re.sub(
r'<a\s+[^>]*href=["\']([^"\']+)["\'][^>]*>([\s\S]*?)</a>',
lambda m: f"[{_strip_tags(m[2])}]({m[1]})",
html,
flags=re.I,
)
text = re.sub(
r"<h([1-6])[^>]*>([\s\S]*?)</h\1>",
lambda m: f"\n{'#' * int(m[1])} {_strip_tags(m[2])}\n",
text,
flags=re.I,
)
text = re.sub(
r"<li[^>]*>([\s\S]*?)</li>", lambda m: f"\n- {_strip_tags(m[1])}", text, flags=re.I
)
text = re.sub(r"</(p|div|section|article)>", "\n\n", text, flags=re.I)
text = re.sub(r"<(br|hr)\s*/?>", "\n", text, flags=re.I)
return _normalize(_strip_tags(text))