Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
from dataclasses import FrozenInstanceError
from types import SimpleNamespace
import pytest
from openviking.parse.accessors.web_importer import (
WebImporter,
WebImportOptions,
parse_web_import_options,
)
from openviking_cli.exceptions import FailedPreconditionError, InvalidArgumentError
class TestParseWebImportOptions:
def test_defaults(self):
opts = parse_web_import_options({})
assert opts.depth == 0
assert opts.max_pages == 50
assert opts.allow_external_links is False
assert opts.skip_download_links is True
assert opts.include_paths is None
assert opts.exclude_paths is None
def test_depth_and_max_pages_parsed(self):
opts = parse_web_import_options({"depth": 2, "max_pages": 30})
assert opts.depth == 2
assert opts.max_pages == 30
def test_max_pages_zero_rejected(self):
with pytest.raises(InvalidArgumentError):
parse_web_import_options({"max_pages": 0})
def test_depth_minus_two_rejected(self):
with pytest.raises(InvalidArgumentError):
parse_web_import_options({"depth": -2})
def test_unlimited_combo_with_external_rejected(self):
with pytest.raises(InvalidArgumentError):
parse_web_import_options(
{"depth": -1, "max_pages": -1, "allow_external_links": True}
)
def test_unlimited_depth_with_bounded_pages_ok(self):
opts = parse_web_import_options(
{"depth": -1, "max_pages": 100, "allow_external_links": True}
)
assert opts.depth == -1
assert opts.max_pages == 100
assert opts.allow_external_links is True
def test_string_int_coerced(self):
opts = parse_web_import_options({"depth": "3"})
assert opts.depth == 3
def test_invalid_int_rejected(self):
with pytest.raises(InvalidArgumentError):
parse_web_import_options({"depth": "abc"})
def test_bool_parsed_from_string(self):
opts = parse_web_import_options({"allow_external_links": "true"})
assert opts.allow_external_links is True
def test_skip_download_links_parsed_from_string(self):
opts = parse_web_import_options({"skip_download_links": "true"})
assert opts.skip_download_links is True
def test_skip_download_links_can_be_disabled(self):
opts = parse_web_import_options({"skip_download_links": "false"})
assert opts.skip_download_links is False
def test_invalid_bool_rejected(self):
with pytest.raises(InvalidArgumentError):
parse_web_import_options({"allow_external_links": "maybe"})
def test_include_paths_csv(self):
opts = parse_web_import_options({"include_paths": "/docs/,/api/"})
assert opts.include_paths == ["/docs/", "/api/"]
def test_include_paths_list(self):
opts = parse_web_import_options({"include_paths": ["/a", "/b"]})
assert opts.include_paths == ["/a", "/b"]
def test_include_paths_empty_drops_to_none(self):
opts = parse_web_import_options({"include_paths": ""})
assert opts.include_paths is None
def test_processor_args_popped_after_parse(self):
args = {"depth": 2, "max_pages": 30, "other": "keep"}
parse_web_import_options(args)
assert args == {"other": "keep"}
class TestWebImportOptionsDataclass:
def test_frozen(self):
opts = WebImportOptions()
with pytest.raises(FrozenInstanceError):
opts.depth = 5 # type: ignore[misc]
class TestWebImporter:
async def test_import_to_directory_writes_html_and_dedupes(self, monkeypatch):
class FakeCrawler:
def __init__(self, config):
self.config = config
async def crawl(self, root_url):
return SimpleNamespace(
pages=[
SimpleNamespace(
url=root_url,
final_url=root_url,
depth=0,
status="success",
html="<html><body><h1>Home</h1></body></html>",
),
SimpleNamespace(
url=f"{root_url}/guide",
final_url=f"{root_url}/guide",
depth=1,
status="success",
html="<html><body><h1>Guide</h1></body></html>",
),
SimpleNamespace(
url=f"{root_url}/guide#frag",
final_url=f"{root_url}/guide#frag",
depth=1,
status="success",
html="<html><body><h1>Duplicate</h1></body></html>",
),
],
downloads=[],
total_crawled=3,
total_downloads=0,
total_failed=0,
total_skipped=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
result = await WebImporter().import_to_directory(
root_url="https://example.com",
options=WebImportOptions(depth=1, max_pages=3),
)
try:
files = sorted(path.relative_to(result.path).as_posix() for path in result.path.rglob("*.html"))
assert files == ["Guide.html", "Home.html"]
assert result.path.name == "example.com"
assert (result.path / "Home.html").read_text(encoding="utf-8") == (
"<html><body><h1>Home</h1></body></html>"
)
assert result.meta["page_count"] == 2
assert result.meta["original_filename"] == "example.com"
finally:
import shutil
shutil.rmtree(result.path.parent, ignore_errors=True)
async def test_import_to_directory_rejects_missing_entry(self, monkeypatch):
class FakeCrawler:
def __init__(self, config):
self.config = config
async def crawl(self, root_url):
return SimpleNamespace(
pages=[
SimpleNamespace(
url=f"{root_url}/child",
final_url=f"{root_url}/child",
depth=1,
status="success",
html="<html>child</html>",
)
],
downloads=[],
total_crawled=1,
total_downloads=0,
total_failed=0,
total_skipped=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
with pytest.raises(RuntimeError, match="Failed to fetch entry page"):
await WebImporter().import_to_directory(
root_url="https://example.com",
options=WebImportOptions(),
)
async def test_entry_failure_surfaces_error_detail(self, monkeypatch):
class FakeCrawler:
def __init__(self, config):
self.config = config
async def crawl(self, root_url):
return SimpleNamespace(
pages=[
SimpleNamespace(
url=root_url,
final_url=root_url,
depth=0,
status="failed",
html=None,
error="blocked by SSRF",
)
],
downloads=[],
total_crawled=0,
total_downloads=0,
total_failed=1,
total_skipped=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
with pytest.raises(RuntimeError, match="blocked by SSRF"):
await WebImporter().import_to_directory(
root_url="https://example.com",
options=WebImportOptions(),
)
async def test_entry_failure_robots_txt_is_humanized(self, monkeypatch):
class FakeCrawler:
def __init__(self, config):
self.config = config
async def crawl(self, root_url):
return SimpleNamespace(
pages=[
SimpleNamespace(
url=root_url,
final_url=root_url,
depth=0,
status="failed",
html=None,
error="Forbidden by robots.txt",
)
],
downloads=[],
total_crawled=0,
total_downloads=0,
total_failed=1,
total_skipped=0,
)
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
with pytest.raises(FailedPreconditionError) as exc_info:
await WebImporter().import_to_directory(
root_url="https://mp.weixin.qq.com/s/abc",
options=WebImportOptions(),
)
message = str(exc_info.value)
assert exc_info.value.code == "FAILED_PRECONDITION"
assert "Forbidden by robots.txt" not in message
assert "compliance" in message
assert "local file" in message
async def test_import_to_directory_downloads_child_links(self, monkeypatch, tmp_path):
class FakeCrawler:
def __init__(self, config):
self.config = config
async def crawl(self, root_url):
return SimpleNamespace(
pages=[
SimpleNamespace(
url=root_url,
final_url=root_url,
depth=0,
status="success",
html="<html><body><h1>Home</h1></body></html>",
)
],
downloads=[
SimpleNamespace(url=f"{root_url}/docs/llms.txt", depth=1),
],
total_crawled=1,
total_downloads=1,
total_failed=0,
total_skipped=0,
)
download_file = tmp_path / "download.txt"
download_file.write_text("plain docs", encoding="utf-8")
async def fake_download_url(self, url, request_validator=None):
return str(download_file), SimpleNamespace(value="download_txt"), {
"original_filename": "llms.txt",
}
monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
monkeypatch.setattr(
"openviking.parse.accessors.http_accessor.HTTPAccessor._download_url",
fake_download_url,
)
result = await WebImporter().import_to_directory(
root_url="https://example.com",
options=WebImportOptions(depth=1, max_pages=2),
)
try:
assert (result.path / "docs" / "llms.txt").read_text(encoding="utf-8") == (
"plain docs"
)
assert result.meta["download_count"] == 1
assert result.meta["crawl_result"]["total_downloads"] == 1
finally:
import shutil
shutil.rmtree(result.path.parent, ignore_errors=True)