Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Unit tests for WebFeedAccessor (whole-site ingestion via sitemap / RSS / Atom)."""
import pytest
from openviking.parse.accessors import (
AccessorRegistry,
GitAccessor,
HTTPAccessor,
WebFeedAccessor,
)
from openviking.parse.accessors.web_feed_accessor import (
FeedEntry,
_classify_and_count,
_entry_to_relpath,
_extract_feed_links,
discover_feed_hint,
looks_like_feed_url,
url_to_relpath,
)
SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
# --------------------------------------------------------------------------- #
# Fake httpx client (dependency-free; no real network)
# --------------------------------------------------------------------------- #
class _FakeStatusError(Exception):
pass
class _FakeResponse:
def __init__(self, status_code=200, content=b"", content_type="text/html"):
self.status_code = status_code
self.content = content if isinstance(content, bytes) else content.encode("utf-8")
self.headers = {"content-type": content_type}
@property
def text(self):
return self.content.decode("utf-8", "replace")
def raise_for_status(self):
if self.status_code >= 400:
raise _FakeStatusError(f"status {self.status_code}")
def _make_fake_client(routes):
"""routes: dict url -> _FakeResponse (missing urls -> 404)."""
class _FakeClient:
def __init__(self, **kwargs):
self.kwargs = kwargs
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
return False
async def get(self, url, headers=None):
resp = routes.get(url)
if resp is None:
return _FakeResponse(404, b"not found", "text/plain")
return resp
return _FakeClient
@pytest.fixture
def patch_httpx(monkeypatch):
"""Patch httpx.AsyncClient with a fake serving the given routes."""
def _apply(routes):
import httpx
monkeypatch.setattr(httpx, "AsyncClient", _make_fake_client(routes))
return _apply
def _urlset(urls_with_lastmod):
items = "".join(
f"<url><loc>{u}</loc>" + (f"<lastmod>{lm}</lastmod>" if lm else "") + "</url>"
for u, lm in urls_with_lastmod
)
return f'<?xml version="1.0"?><urlset xmlns="{SITEMAP_NS}">{items}</urlset>'.encode()
def _sitemapindex(sub_urls):
items = "".join(f"<sitemap><loc>{u}</loc></sitemap>" for u in sub_urls)
return (
f'<?xml version="1.0"?><sitemapindex xmlns="{SITEMAP_NS}">{items}</sitemapindex>'.encode()
)
def _page(title):
return f"<html><head><title>{title}</title></head><body><h1>{title}</h1><p>body</p></body></html>".encode()
# --------------------------------------------------------------------------- #
# can_handle
# --------------------------------------------------------------------------- #
class TestCanHandle:
@pytest.fixture
def accessor(self):
return WebFeedAccessor()
def test_priority(self, accessor):
assert accessor.priority == 60
@pytest.mark.parametrize(
"url",
[
"https://blog.openviking.ai/sitemap.xml",
"https://t0saki.com/sitemap-index.xml",
"https://t0saki.com/sitemap_index.xml",
"https://t0saki.com/sitemap-0.xml",
"https://t0saki.com/rss.xml",
"https://example.com/feed.xml",
"https://example.com/atom.xml",
"https://example.com/index.xml",
"https://example.com/feed",
"https://example.com/blog/feed/",
],
)
def test_handles_feed_like(self, accessor, url):
assert accessor.can_handle(url) is True
@pytest.mark.parametrize(
"url",
[
"https://example.com/post/foo/",
"https://example.com/data.xml",
"https://example.com/page.html",
"/tmp/sitemap.xml",
"[email protected]:org/repo.git",
],
)
def test_rejects_non_feed(self, accessor, url):
assert accessor.can_handle(url) is False
def test_explicit_override(self, accessor):
assert accessor.can_handle("https://example.com/post/foo/", site=True) is True
assert accessor.can_handle("https://example.com/sitemap.xml", site=False) is False
# alias keys
assert accessor.can_handle("https://example.com/x", sitemap=True) is True
assert accessor.can_handle("https://example.com/x", feed=True) is True
# --------------------------------------------------------------------------- #
# registry routing
# --------------------------------------------------------------------------- #
class TestRegistryRouting:
@pytest.fixture
def registry(self):
return AccessorRegistry()
def test_sitemap_routes_to_web_feed(self, registry):
acc = registry.get_accessor("https://blog.openviking.ai/sitemap.xml")
assert isinstance(acc, WebFeedAccessor)
def test_git_url_still_wins(self, registry):
acc = registry.get_accessor("https://github.com/volcengine/OpenViking")
assert isinstance(acc, GitAccessor)
def test_plain_page_routes_to_http(self, registry):
acc = registry.get_accessor("https://example.com/article")
assert isinstance(acc, HTTPAccessor)
def test_override_routes_plain_page_to_web_feed(self, registry):
acc = registry.get_accessor("https://example.com/article", site=True)
assert isinstance(acc, WebFeedAccessor)
# --------------------------------------------------------------------------- #
# url -> relpath
# --------------------------------------------------------------------------- #
class TestRelpath:
@pytest.mark.parametrize(
"url,expected",
[
("https://h/", "index.html"),
("https://h/a/b", "a/b.html"),
("https://h/post/foo/", "post/foo.html"),
("https://h/post/foo/index.html", "post/foo/index.html"),
("https://h/about.php", "about.html"),
],
)
def test_url_to_relpath(self, url, expected):
assert url_to_relpath(url) == expected
def test_entry_title_used_when_present(self):
assert (
_entry_to_relpath(FeedEntry(url="https://h/x", title="Hello World"))
== "Hello World.html"
)
def test_entry_title_cjk(self):
assert _entry_to_relpath(FeedEntry(url="https://h/x", title="台湾游记")) == "台湾游记.html"
def test_entry_no_title_falls_back_to_url(self):
assert _entry_to_relpath(FeedEntry(url="https://h/post/bar/")) == "post/bar.html"
def test_title_with_slash_is_sanitized(self):
rel = _entry_to_relpath(FeedEntry(url="https://h/x", title="a/b: c"))
assert "/" not in rel[:-5] # no path separator in the stem
# --------------------------------------------------------------------------- #
# classify-and-count (used by the single-page hint)
# --------------------------------------------------------------------------- #
class TestClassify:
def test_urlset(self):
content = _urlset([("https://h/a", None), ("https://h/b", "2026-01-01")])
assert _classify_and_count(content) == ("sitemap", 2)
def test_sitemapindex(self):
assert _classify_and_count(_sitemapindex(["https://h/s0.xml"])) == ("sitemapindex", 1)
def test_rss(self):
rss = b'<?xml version="1.0"?><rss version="2.0"><channel><item><link>https://h/a</link></item></channel></rss>'
assert _classify_and_count(rss) == ("rss", 1)
def test_garbage_returns_none(self):
assert _classify_and_count(b"not xml at all") is None
def test_extract_feed_links(self):
html = (
'<link rel="alternate" type="application/rss+xml" href="/rss.xml">'
'<link rel="alternate" type="application/atom+xml" href="https://h/atom.xml">'
'<link rel="sitemap" href="/sitemap.xml">'
'<link rel="stylesheet" href="/x.css">'
)
links = _extract_feed_links(html, "https://h/page")
assert "https://h/rss.xml" in links
assert "https://h/atom.xml" in links
assert "https://h/sitemap.xml" in links
assert all("x.css" not in link for link in links)
# --------------------------------------------------------------------------- #
# access() — sitemap mirroring
# --------------------------------------------------------------------------- #
class TestAccessSitemap:
async def test_flat_urlset_mirrors_pages(self, patch_httpx):
base = "https://blog.example.com"
page_urls = [f"{base}/post/p{i}/" for i in range(10)]
routes = {
f"{base}/sitemap.xml": _FakeResponse(content=_urlset([(u, None) for u in page_urls]))
}
for i, u in enumerate(page_urls):
routes[u] = _FakeResponse(content=_page(f"Post {i}"))
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(f"{base}/sitemap.xml", respect_robots=False, politeness_delay=0)
try:
assert resource.is_temporary is True
assert resource.path.is_dir()
assert resource.meta["original_filename"] == "blog.example.com"
assert resource.meta["page_count"] == 10
assert resource.meta["url_type"] == "sitemap"
html_files = list(resource.path.rglob("*.html"))
assert len(html_files) == 10
# mirrors URL path structure (post/p0.html ...)
assert (resource.path / "post" / "p0.html").exists()
finally:
resource.cleanup()
async def test_failed_page_is_skipped(self, patch_httpx):
base = "https://blog.example.com"
page_urls = [f"{base}/post/p{i}/" for i in range(5)]
routes = {
f"{base}/sitemap.xml": _FakeResponse(content=_urlset([(u, None) for u in page_urls]))
}
for i, u in enumerate(page_urls):
routes[u] = _FakeResponse(content=_page(f"Post {i}"))
routes[page_urls[2]] = _FakeResponse(500, b"err", "text/plain") # one fails
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(f"{base}/sitemap.xml", respect_robots=False, politeness_delay=0)
try:
assert resource.meta["page_count"] == 4
finally:
resource.cleanup()
async def test_sitemapindex_recursion(self, patch_httpx):
base = "https://t0saki.com"
page_urls = [f"{base}/posts/p{i}/" for i in range(3)]
routes = {
f"{base}/sitemap-index.xml": _FakeResponse(
content=_sitemapindex([f"{base}/sitemap-0.xml"])
),
f"{base}/sitemap-0.xml": _FakeResponse(content=_urlset([(u, None) for u in page_urls])),
}
for i, u in enumerate(page_urls):
routes[u] = _FakeResponse(content=_page(f"P{i}"))
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(
f"{base}/sitemap-index.xml", respect_robots=False, politeness_delay=0
)
try:
assert resource.meta["page_count"] == 3
assert resource.meta["url_type"] == "sitemap"
finally:
resource.cleanup()
async def test_max_pages_truncates(self, patch_httpx):
base = "https://blog.example.com"
page_urls = [f"{base}/p{i}/" for i in range(10)]
routes = {
f"{base}/sitemap.xml": _FakeResponse(content=_urlset([(u, None) for u in page_urls]))
}
for u in page_urls:
routes[u] = _FakeResponse(content=_page("x"))
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(
f"{base}/sitemap.xml", respect_robots=False, politeness_delay=0, max_pages=3
)
try:
assert resource.meta["page_count"] == 3
finally:
resource.cleanup()
async def test_same_host_only_filters_external(self, patch_httpx):
base = "https://blog.example.com"
internal = f"{base}/keep/"
external = "https://other.com/drop/"
routes = {
f"{base}/sitemap.xml": _FakeResponse(
content=_urlset([(internal, None), (external, None)])
),
internal: _FakeResponse(content=_page("keep")),
external: _FakeResponse(content=_page("drop")),
}
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(f"{base}/sitemap.xml", respect_robots=False, politeness_delay=0)
try:
assert resource.meta["page_count"] == 1
assert internal in resource.meta["pages"]
assert external not in resource.meta["pages"]
finally:
resource.cleanup()
# --------------------------------------------------------------------------- #
# access() — auto-discovery from a bare domain / HTML page (args={"site": True})
# --------------------------------------------------------------------------- #
class TestAutoDiscovery:
async def test_bare_domain_discovers_sitemap_via_robots(self, patch_httpx):
base = "https://t0saki.example"
page_urls = [f"{base}/posts/p{i}/" for i in range(3)]
routes = {
base: _FakeResponse(content=b"<html><body>home</body></html>"),
f"{base}/robots.txt": _FakeResponse(
content=f"User-agent: *\nSitemap: {base}/sitemap.xml\n", content_type="text/plain"
),
f"{base}/sitemap.xml": _FakeResponse(content=_urlset([(u, None) for u in page_urls])),
}
for i, u in enumerate(page_urls):
routes[u] = _FakeResponse(content=_page(f"P{i}"))
patch_httpx(routes)
acc = WebFeedAccessor()
# access() simulates the args={"site": True} routing (can_handle override).
resource = await acc.access(base, respect_robots=False, politeness_delay=0)
try:
assert resource.meta["page_count"] == 3
assert resource.meta["feed_url"] == f"{base}/sitemap.xml"
assert resource.meta["original_filename"] == "t0saki.example"
finally:
resource.cleanup()
async def test_html_page_discovers_feed_via_link_tag(self, patch_httpx):
base = "https://blog.example"
page_html = (
"<html><head>"
f'<link rel="alternate" type="application/rss+xml" href="{base}/rss.xml">'
"</head><body>post</body></html>"
).encode()
rss = (
'<?xml version="1.0"?><rss version="2.0"><channel>'
f"<item><title>One</title><link>{base}/a/</link></item>"
"</channel></rss>"
).encode()
routes = {
f"{base}/some-article": _FakeResponse(content=page_html),
f"{base}/rss.xml": _FakeResponse(content=rss, content_type="application/rss+xml"),
f"{base}/a/": _FakeResponse(content=_page("A")),
}
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(
f"{base}/some-article", respect_robots=False, politeness_delay=0
)
try:
assert resource.meta["url_type"] == "rss"
assert resource.meta["feed_url"] == f"{base}/rss.xml"
finally:
resource.cleanup()
async def test_no_feed_discoverable_raises(self, patch_httpx):
base = "https://nofeed.example"
routes = {base: _FakeResponse(content=b"<html><body>nothing</body></html>")}
patch_httpx(routes)
acc = WebFeedAccessor()
with pytest.raises(RuntimeError, match="auto-discovered"):
await acc.access(base, respect_robots=False, politeness_delay=0)
# --------------------------------------------------------------------------- #
# access() — RSS / Atom
# --------------------------------------------------------------------------- #
class TestAccessFeed:
async def test_rss_summary_only_fetches_links(self, patch_httpx):
base = "https://t0saki.com"
items = [(f"{base}/posts/a/", "Post A"), (f"{base}/posts/b/", "Post B")]
rss = (
'<?xml version="1.0"?><rss version="2.0"><channel>'
+ "".join(
f"<item><title>{t}</title><link>{u}</link><description>summary</description></item>"
for u, t in items
)
+ "</channel></rss>"
).encode()
routes = {f"{base}/rss.xml": _FakeResponse(content=rss, content_type="application/rss+xml")}
for u, _ in items:
routes[u] = _FakeResponse(content=_page("full"))
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(f"{base}/rss.xml", respect_robots=False, politeness_delay=0)
try:
assert resource.meta["url_type"] == "rss"
assert resource.meta["page_count"] == 2
# titles used as flat filenames
assert (resource.path / "Post A.html").exists()
assert (resource.path / "Post B.html").exists()
finally:
resource.cleanup()
async def test_rss_inline_content_is_used_without_fetch(self, patch_httpx):
base = "https://t0saki.com"
rss = (
'<?xml version="1.0"?>'
'<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">'
"<channel><item><title>Inline</title><link>https://t0saki.com/posts/inline/</link>"
"<content:encoded><![CDATA[<p>FULL INLINE BODY</p>]]></content:encoded>"
"</item></channel></rss>"
).encode()
# Note: deliberately DO NOT register the article URL — if access tried to
# fetch it we'd get a 404 and page_count would drop to 0.
routes = {f"{base}/rss.xml": _FakeResponse(content=rss, content_type="application/rss+xml")}
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(f"{base}/rss.xml", respect_robots=False, politeness_delay=0)
try:
assert resource.meta["page_count"] == 1
saved = (resource.path / "Inline.html").read_text(encoding="utf-8")
assert "FULL INLINE BODY" in saved
finally:
resource.cleanup()
async def test_atom_feed(self, patch_httpx):
base = "https://example.org"
atom = (
'<?xml version="1.0"?><feed xmlns="http://www.w3.org/2005/Atom">'
'<entry><title>Entry One</title><link href="https://example.org/e1/"/></entry>'
"</feed>"
).encode()
routes = {
f"{base}/atom.xml": _FakeResponse(content=atom, content_type="application/atom+xml"),
f"{base}/e1/": _FakeResponse(content=_page("E1")),
}
patch_httpx(routes)
acc = WebFeedAccessor()
resource = await acc.access(f"{base}/atom.xml", respect_robots=False, politeness_delay=0)
try:
assert resource.meta["url_type"] == "atom"
assert resource.meta["page_count"] == 1
finally:
resource.cleanup()
# --------------------------------------------------------------------------- #
# discover_feed_hint (single-page detect-and-suggest)
# --------------------------------------------------------------------------- #
class TestDiscoverHint:
async def test_hint_from_robots_sitemap(self, patch_httpx):
base = "https://blog.example.com"
routes = {
f"{base}/robots.txt": _FakeResponse(
content=f"User-agent: *\nSitemap: {base}/sitemap.xml\n", content_type="text/plain"
),
f"{base}/sitemap.xml": _FakeResponse(
content=_urlset([(f"{base}/p{i}/", None) for i in range(3)])
),
f"{base}/article": _FakeResponse(content=_page("article")),
}
patch_httpx(routes)
hint = await discover_feed_hint(f"{base}/article", timeout=5)
assert hint is not None
assert "3 page(s)" in hint
assert f"{base}/sitemap.xml" in hint
async def test_hint_from_html_autodiscovery(self, patch_httpx):
base = "https://blog.example.com"
page_html = (
'<html><head><link rel="alternate" type="application/rss+xml" '
f'href="{base}/rss.xml"></head><body>hi</body></html>'
).encode()
rss = (
'<?xml version="1.0"?><rss version="2.0"><channel>'
"<item><link>https://blog.example.com/a/</link></item>"
"<item><link>https://blog.example.com/b/</link></item>"
"</channel></rss>"
).encode()
routes = {
f"{base}/article": _FakeResponse(content=page_html),
f"{base}/rss.xml": _FakeResponse(content=rss, content_type="application/rss+xml"),
# robots.txt missing -> 404, fine
}
patch_httpx(routes)
hint = await discover_feed_hint(f"{base}/article", timeout=5)
assert hint is not None
assert "RSS feed with 2 item(s)" in hint
async def test_no_hint_when_nothing_found(self, patch_httpx):
base = "https://nofeed.example.com"
routes = {f"{base}/article": _FakeResponse(content=_page("article"))}
patch_httpx(routes)
assert await discover_feed_hint(f"{base}/article", timeout=5) is None
async def test_no_hint_for_feed_url_itself(self, patch_httpx):
patch_httpx({})
assert await discover_feed_hint("https://x.com/sitemap.xml", timeout=5) is None
async def test_no_hint_for_non_html_extension(self, patch_httpx):
patch_httpx({})
assert await discover_feed_hint("https://x.com/file.pdf", timeout=5) is None
# --------------------------------------------------------------------------- #
# looks_like_feed_url (standalone)
# --------------------------------------------------------------------------- #
def test_looks_like_feed_url():
assert looks_like_feed_url("https://h/sitemap.xml")
assert looks_like_feed_url("https://h/rss.xml")
assert not looks_like_feed_url("https://h/post")
assert not looks_like_feed_url("/tmp/sitemap.xml")