Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""Unit tests for HTTPAccessor."""
import socket
from types import SimpleNamespace
from unittest.mock import patch
import httpx
import pytest
from openviking.parse.accessors import AccessorRegistry, GitAccessor, HTTPAccessor
from openviking.parse.accessors.http_accessor import URLType
from openviking.server.models import ERROR_CODE_TO_HTTP_STATUS
from openviking_cli.exceptions import OpenVikingError
def _mock_config():
return SimpleNamespace(
code=SimpleNamespace(
github_domains=["github.com", "www.github.com"],
gitlab_domains=["gitlab.com", "www.gitlab.com"],
github_raw_domain="raw.githubusercontent.com",
azure_devops_domains=[
"dev.azure.com",
"ssh.dev.azure.com",
"vs-ssh.visualstudio.com",
],
code_hosting_domains=["github.com", "gitlab.com"],
)
)
class TestHTTPAccessor:
"""Tests for HTTPAccessor."""
@pytest.fixture(autouse=True)
def _patch_config(self):
with patch(
"openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
side_effect=_mock_config,
):
yield
@pytest.fixture
def accessor(self) -> HTTPAccessor:
"""Create a HTTPAccessor instance."""
return HTTPAccessor()
@pytest.mark.parametrize(
("failure", "expected_code", "expected_status"),
[
(socket.gaierror(socket.EAI_NONAME, "host does not exist"), "INVALID_ARGUMENT", 400),
(socket.gaierror(socket.EAI_AGAIN, "temporary DNS failure"), "UNAVAILABLE", 503),
(httpx.ConnectError("connection refused"), "UNAVAILABLE", 503),
(httpx.ReadTimeout("read timed out"), "DEADLINE_EXCEEDED", 504),
(httpx.InvalidURL("invalid host"), "INVALID_ARGUMENT", 400),
(403, "PERMISSION_DENIED", 403),
(500, "UNAVAILABLE", 503),
],
)
async def test_source_fetch_errors_preserve_cause(
self, accessor: HTTPAccessor, monkeypatch, failure, expected_code, expected_status
) -> None:
async def handler(request):
if isinstance(failure, socket.gaierror):
raise httpx.ConnectError(str(failure)) from failure
if isinstance(failure, Exception):
raise failure
return httpx.Response(failure, request=request)
client_type = httpx.AsyncClient
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: client_type(transport=httpx.MockTransport(handler), **kwargs),
)
with pytest.raises(OpenVikingError) as caught:
await accessor.access("https://source.example/data.txt")
assert caught.value.code == expected_code
assert ERROR_CODE_TO_HTTP_STATUS[caught.value.code] == expected_status
@pytest.mark.parametrize(
"source",
[
"https://example.com/page.html",
"http://example.com/document.pdf",
"https://example.org/file.md",
],
)
def test_can_handle_http_urls(self, accessor: HTTPAccessor, source: str) -> None:
"""HTTPAccessor should handle regular HTTP/HTTPS URLs."""
assert accessor.can_handle(source) is True
@pytest.mark.parametrize(
"source",
[
"/path/to/file.html",
"[email protected]:org/repo.git",
"plain text content",
],
)
def test_cannot_handle_non_http(self, accessor: HTTPAccessor, source: str) -> None:
"""HTTPAccessor should NOT handle non-HTTP sources."""
assert accessor.can_handle(source) is False
@pytest.mark.parametrize(
"url, expected",
[
("https://example.com/path/file.html", "file.html"),
("https://example.com/path/doc.pdf", "doc.pdf"),
("https://example.com/path/", "path"),
("https://example.com", "download"),
],
)
def test_extract_filename_from_url(self, url: str, expected: str) -> None:
"""Test filename extraction from URLs."""
assert HTTPAccessor._extract_filename_from_url(url) == expected
@pytest.mark.parametrize(
("tos_args", "header_name", "header_value"),
[
({"tos_signature": "signed-value"}, "X-Tos-Signature", "signed-value"),
({"tos_access": "access-key"}, "X-Tos-Access", "access-key"),
],
)
async def test_tos_auth_is_sent_for_head_and_get(
self, accessor: HTTPAccessor, monkeypatch: pytest.MonkeyPatch, tos_args, header_name, header_value
) -> None:
seen: list[tuple[str, str | None]] = []
async def handler(request: httpx.Request) -> httpx.Response:
seen.append((request.method, request.headers.get(header_name)))
return httpx.Response(
200,
headers={"Content-Type": "text/plain"},
content=b"TOS resource",
request=request,
)
class Client:
def __init__(self, **_kwargs):
self._client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
async def __aenter__(self):
return self._client
async def __aexit__(self, *args):
await self._client.aclose()
monkeypatch.setattr(
"openviking.parse.accessors.http_accessor.lazy_import",
lambda _name: SimpleNamespace(AsyncClient=Client),
)
resource = await accessor.access(
"https://tos.example.com/object",
**tos_args,
)
try:
assert seen == [("HEAD", header_value), ("GET", header_value)]
finally:
resource.cleanup()
async def test_tos_auth_args_do_not_reach_parser(
self, tmp_path, monkeypatch: pytest.MonkeyPatch
) -> None:
from openviking.parse.accessors.base import LocalResource, SourceType
from openviking.utils.media_processor import UnifiedResourceProcessor
source = tmp_path / "resource.txt"
source.write_text("content", encoding="utf-8")
local = LocalResource(
path=source,
source_type=SourceType.HTTP,
original_source="https://tos.example.com/object",
is_temporary=False,
)
received: dict[str, object] = {}
class Registry:
async def access(self, *_args, **_kwargs):
return local
class Router:
def should_use_understanding_directly(self, *_args, **_kwargs):
return False
async def parse(self, _source, **kwargs):
received.update(kwargs)
return SimpleNamespace()
processor = UnifiedResourceProcessor()
processor._accessor_registry = Registry()
processor._parser_router = Router()
monkeypatch.setattr(processor, "_get_vlm_processor", lambda: None)
await processor.process(
"https://tos.example.com/object",
tos_signature="signed-value",
tos_access="access-key",
)
assert "tos_signature" not in received
assert "tos_access" not in received
async def test_tos_auth_args_do_not_reach_direct_understanding_parser(
self, monkeypatch: pytest.MonkeyPatch
) -> None:
from openviking.utils.media_processor import UnifiedResourceProcessor
received: dict[str, object] = {}
class Router:
def should_use_understanding_directly(self, *_args, **_kwargs):
return True
async def parse(self, _source, **kwargs):
received.update(kwargs)
return SimpleNamespace()
processor = UnifiedResourceProcessor()
processor._parser_router = Router()
monkeypatch.setattr(processor, "_get_vlm_processor", lambda: None)
await processor.process(
"https://tos.example.com/object",
tos_signature="signed-value",
tos_access="access-key",
)
assert "tos_signature" not in received
assert "tos_access" not in received
@pytest.mark.parametrize("entry_url_type", [URLType.WEBPAGE, URLType.DOWNLOAD_HTML])
async def test_webpage_uses_web_importer_directory(
self, accessor, tmp_path, monkeypatch, entry_url_type
):
downloaded = tmp_path / "entry.html"
downloaded.write_text("<html>entry</html>", encoding="utf-8")
imported_dir = tmp_path / "web"
imported_dir.mkdir()
async def fake_download(url, request_validator=None):
return str(downloaded), entry_url_type, {"extension": ".html"}
class FakeImporter:
async def import_to_directory(self, *, root_url, options, request_validator=None):
assert root_url == "https://example.com/page"
assert options.depth == 1
return SimpleNamespace(
path=imported_dir,
meta={
"web_import": True,
"crawl_result": {"total_crawled": 1},
"original_filename": "example.com",
},
)
monkeypatch.setattr(accessor, "_download_url", fake_download)
monkeypatch.setattr(
"openviking.parse.accessors.web_importer.WebImporter",
lambda: FakeImporter(),
)
resource = await accessor.access("https://example.com/page", depth=1)
assert resource.path == imported_dir
assert resource.path.is_dir()
assert resource.meta["url_type"] == "webpage"
assert resource.meta["web_import"] is True
assert not downloaded.exists()
async def test_download_file_does_not_use_web_importer(self, accessor, tmp_path, monkeypatch):
downloaded = tmp_path / "file.pdf"
downloaded.write_bytes(b"%PDF-test")
called = False
async def fake_download(url, request_validator=None):
return str(downloaded), URLType.DOWNLOAD_PDF, {"extension": ".pdf"}
class FakeImporter:
async def import_to_directory(self, **kwargs):
nonlocal called
called = True
monkeypatch.setattr(accessor, "_download_url", fake_download)
monkeypatch.setattr(
"openviking.parse.accessors.web_importer.WebImporter",
lambda: FakeImporter(),
)
resource = await accessor.access("https://example.com/file.pdf", depth=1)
assert resource.path == downloaded
assert resource.meta["url_type"] == "download_pdf"
assert called is False
@pytest.mark.parametrize(
"url",
[
"https://github.com/org/repo/blob/main/x.html",
"https://raw.githubusercontent.com/org/repo/main/x.html",
"https://gitlab.com/org/repo/blob/main/x.html",
],
)
async def test_code_hosting_single_file_does_not_use_web_importer(
self, accessor, tmp_path, monkeypatch, url
):
"""Code-hosting single-file URLs (blob/raw) stay on the single-file path.
Even though the download resolves to an ``.html`` file (DOWNLOAD_HTML),
these URLs are semantically one file, not a crawlable site, so they must
NOT be routed through WebImporter.
"""
downloaded = tmp_path / "x.html"
downloaded.write_text("<html>file</html>", encoding="utf-8")
called = False
async def fake_download(u, request_validator=None):
return str(downloaded), URLType.DOWNLOAD_HTML, {"extension": ".html"}
class FakeImporter:
async def import_to_directory(self, **kwargs):
nonlocal called
called = True
monkeypatch.setattr(accessor, "_download_url", fake_download)
monkeypatch.setattr(
"openviking.parse.accessors.web_importer.WebImporter",
lambda: FakeImporter(),
)
resource = await accessor.access(url, depth=1)
assert resource.path == downloaded
assert resource.meta["url_type"] == "download_html"
assert called is False
class TestHTTPAccessorPriorityRouting:
"""Tests that verify HTTPAccessor works correctly with priority-based routing."""
@pytest.fixture(autouse=True)
def _patch_config(self):
with patch(
"openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
side_effect=_mock_config,
):
yield
def test_git_url_routed_to_git_accessor(self) -> None:
"""Git URLs should be routed to GitAccessor, not HTTPAccessor."""
registry = AccessorRegistry(register_default=False)
http = HTTPAccessor()
git = GitAccessor()
registry.register(http)
registry.register(git)
test_url = "https://github.com/volcengine/OpenViking"
# Both can handle the URL individually (this is OK!)
assert git.can_handle(test_url) is True
assert http.can_handle(test_url) is True
# But registry picks the higher priority one (GitAccessor)
accessor = registry.get_accessor(test_url)
assert accessor is not None
assert accessor.__class__.__name__ == "GitAccessor"
def test_azure_devops_git_url_routed_to_git_accessor(self) -> None:
"""Azure DevOps repo URLs should be routed to GitAccessor."""
registry = AccessorRegistry(register_default=False)
http = HTTPAccessor()
git = GitAccessor()
registry.register(http)
registry.register(git)
test_url = "https://dev.azure.com/org/project/_git/repo"
assert git.can_handle(test_url) is True
assert http.can_handle(test_url) is True
accessor = registry.get_accessor(test_url)
assert accessor is not None
assert accessor.__class__.__name__ == "GitAccessor"
def test_regular_http_url_routed_to_http_accessor(self) -> None:
"""Regular HTTP URLs should be routed to HTTPAccessor."""
registry = AccessorRegistry(register_default=False)
http = HTTPAccessor()
git = GitAccessor()
registry.register(http)
registry.register(git)
test_url = "https://example.com/page.html"
# Only HTTPAccessor can handle this
assert git.can_handle(test_url) is False
assert http.can_handle(test_url) is True
# Registry picks HTTPAccessor
accessor = registry.get_accessor(test_url)
assert accessor is not None
assert accessor.__class__.__name__ == "HTTPAccessor"
def test_azure_devops_browse_url_routed_to_http_accessor(self) -> None:
"""Azure DevOps browse URLs should stay with HTTPAccessor."""
registry = AccessorRegistry(register_default=False)
http = HTTPAccessor()
git = GitAccessor()
registry.register(http)
registry.register(git)
test_url = "https://dev.azure.com/org/project/_git/repo?path=/README.md"
assert git.can_handle(test_url) is False
assert http.can_handle(test_url) is True
accessor = registry.get_accessor(test_url)
assert accessor is not None
assert accessor.__class__.__name__ == "HTTPAccessor"
class TestHTTPAccessorRawUrlConversion:
"""Tests for HTTPAccessor._convert_to_raw_url (GitHub/GitLab blob -> raw)."""
@pytest.fixture(autouse=True)
def _patch_config(self):
with patch(
"openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
side_effect=_mock_config,
):
yield
@pytest.fixture
def accessor(self) -> HTTPAccessor:
return HTTPAccessor()
def test_github_blob_conversion(self, accessor: HTTPAccessor) -> None:
blob_url = "https://github.com/volcengine/OpenViking/blob/main/docs/design.md"
expected = "https://raw.githubusercontent.com/volcengine/OpenViking/main/docs/design.md"
assert accessor._convert_to_raw_url(blob_url) == expected
blob_deep = "https://github.com/user/repo/blob/feature/branch/src/components/Button.tsx"
expected_deep = (
"https://raw.githubusercontent.com/user/repo/feature/branch/src/components/Button.tsx"
)
assert accessor._convert_to_raw_url(blob_deep) == expected_deep
def test_github_non_blob_urls(self, accessor: HTTPAccessor) -> None:
repo_root = "https://github.com/volcengine/OpenViking"
assert accessor._convert_to_raw_url(repo_root) == repo_root
issue_url = "https://github.com/volcengine/OpenViking/issues/1"
assert accessor._convert_to_raw_url(issue_url) == issue_url
raw_url = "https://raw.githubusercontent.com/volcengine/OpenViking/main/README.md"
assert accessor._convert_to_raw_url(raw_url) == raw_url
def test_gitlab_blob_conversion(self, accessor: HTTPAccessor) -> None:
blob_url = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/README.md"
expected = "https://gitlab.com/gitlab-org/gitlab/-/raw/master/README.md"
assert accessor._convert_to_raw_url(blob_url) == expected
blob_deep = "https://gitlab.com/group/project/-/blob/dev/src/main.rs"
expected_deep = "https://gitlab.com/group/project/-/raw/dev/src/main.rs"
assert accessor._convert_to_raw_url(blob_deep) == expected_deep
def test_gitlab_non_blob_urls(self, accessor: HTTPAccessor) -> None:
root = "https://gitlab.com/gitlab-org/gitlab"
assert accessor._convert_to_raw_url(root) == root
issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/123"
assert accessor._convert_to_raw_url(issue) == issue
def test_other_domains(self, accessor: HTTPAccessor) -> None:
url = "https://example.com/blob/main/file.txt"
assert accessor._convert_to_raw_url(url) == url
bitbucket = "https://bitbucket.org/user/repo/src/master/README.md"
assert accessor._convert_to_raw_url(bitbucket) == bitbucket