Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix security: feat(resources): harden HTTP resource ingestion against private-network SSRF (#1133)
13ernkastel41e345896d24
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.pyindex f1fc737e9..97d3ece62 100644--- a/openviking/parse/parsers/html.py+++ b/openviking/parse/parsers/html.py@@ -29,6 +29,8 @@ from openviking.parse.base import ()from openviking.parse.parsers.base_parser import BaseParserfrom openviking.parse.parsers.constants import CODE_EXTENSIONS+from openviking.utils.network_guard import build_httpx_request_validation_hooks+from openviking_cli.exceptions import PermissionDeniedErrorfrom openviking_cli.utils.config import get_openviking_config@@ -77,7 +79,12 @@ class URLTypeDetector:"application/xhtml+xml": URLType.WEBPAGE,}- async def detect(self, url: str, timeout: float = 10.0) -> Tuple[URLType, Dict[str, Any]]:+ async def detect(+ self,+ url: str,+ timeout: float = 10.0,+ request_validator=None,+ ) -> Tuple[URLType, Dict[str, Any]]:"""Detect URL content type.@@ -107,7 +114,16 @@ class URLTypeDetector:# 2. Send HEAD request to check Content-Typetry:httpx = lazy_import("httpx")- async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:+ client_kwargs = {+ "timeout": timeout,+ "follow_redirects": True,+ }+ event_hooks = build_httpx_request_validation_hooks(request_validator)+ if event_hooks:+ client_kwargs["event_hooks"] = event_hooks+ client_kwargs["trust_env"] = False++ async with httpx.AsyncClient(**client_kwargs) as client:response = await client.head(url)content_type = response.headers.get("content-type", "").lower()@@ -128,6 +144,8 @@ class URLTypeDetector:if "html" in content_type or "xml" in content_type:return URLType.WEBPAGE, meta+ except PermissionDeniedError:+ raiseexcept Exception as e:meta["detection_error"] = str(e)@@ -271,7 +289,12 @@ class HTMLParser(BaseParser):ParseResult"""# Detect URL type- url_type, meta = await self._url_detector.detect(url, timeout=self.timeout)+ request_validator = kwargs.get("request_validator")+ url_type, meta = await self._url_detector.detect(+ url,+ timeout=self.timeout,+ request_validator=request_validator,+ )if url_type == URLType.WEBPAGE:# Fetch and parse as webpage@@ -317,7 +340,10 @@ class HTMLParser(BaseParser):"""try:# Fetch HTML- html_content = await self._fetch_html(url)+ html_content = await self._fetch_html(+ url,+ request_validator=kwargs.get("request_validator"),+ )# Convert to Markdownmarkdown_content = self._html_to_markdown(html_content, base_url=url)@@ -339,6 +365,8 @@ class HTMLParser(BaseParser):return result+ except PermissionDeniedError:+ raiseexcept Exception as e:return create_parse_result(root=ResourceNode(type=NodeType.ROOT, content_path=None),@@ -385,7 +413,10 @@ class HTMLParser(BaseParser):temp_path = Nonetry:# Download to temporary file- temp_path = await self._download_file(url)+ temp_path = await self._download_file(+ url,+ request_validator=kwargs.get("request_validator"),+ )# Extract original filename from URL for use as source_path,# so parsers use it instead of the temp file name.@@ -422,6 +453,8 @@ class HTMLParser(BaseParser):result.meta["url_type"] = f"download_{file_type}"return result+ except PermissionDeniedError:+ raiseexcept Exception as e:return create_parse_result(root=ResourceNode(type=NodeType.ROOT, content_path=None),@@ -457,6 +490,8 @@ class HTMLParser(BaseParser):return result+ except PermissionDeniedError:+ raiseexcept Exception as e:return create_parse_result(root=ResourceNode(type=NodeType.ROOT, content_path=None),@@ -499,7 +534,7 @@ class HTMLParser(BaseParser):warnings=[f"Failed to read HTML: {e}"],)- async def _fetch_html(self, url: str) -> str:+ async def _fetch_html(self, url: str, request_validator=None) -> str:"""Fetch HTML content from URL.@@ -514,7 +549,16 @@ class HTMLParser(BaseParser):"""httpx = lazy_import("httpx")- async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:+ client_kwargs = {+ "timeout": self.timeout,+ "follow_redirects": True,+ }+ event_hooks = build_httpx_request_validation_hooks(request_validator)+ if event_hooks:+ client_kwargs["event_hooks"] = event_hooks+ client_kwargs["trust_env"] = False++ async with httpx.AsyncClient(**client_kwargs) as client:headers = {"User-Agent": self.user_agent}response = await client.get(url, headers=headers)response.raise_for_status()@@ -591,7 +635,7 @@ class HTMLParser(BaseParser):result.temp_dir_path = temp_urireturn result- async def _download_file(self, url: str) -> str:+ async def _download_file(self, url: str, request_validator=None) -> str:"""Download file from URL to temporary location.@@ -619,7 +663,16 @@ class HTMLParser(BaseParser):temp_file.close()# Download- async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client:+ client_kwargs = {+ "timeout": self.timeout,+ "follow_redirects": True,+ }+ event_hooks = build_httpx_request_validation_hooks(request_validator)+ if event_hooks:+ client_kwargs["event_hooks"] = event_hooks+ client_kwargs["trust_env"] = False++ async with httpx.AsyncClient(**client_kwargs) as client:headers = {"User-Agent": self.user_agent}response = await client.get(url, headers=headers)response.raise_for_status()diff --git a/openviking/server/local_input_guard.py b/openviking/server/local_input_guard.pyindex ee0e2d091..d7a08a360 100644--- a/openviking/server/local_input_guard.py+++ b/openviking/server/local_input_guard.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport refrom pathlib import Path+from openviking.utils.network_guard import ensure_public_remote_targetfrom openviking_cli.exceptions import PermissionDeniedError_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:[\\/]")@@ -37,6 +38,7 @@ def require_remote_resource_source(source: str) -> str:"HTTP server only accepts remote resource URLs or temp-uploaded files; ""direct host filesystem paths are not allowed.")+ ensure_public_remote_target(source)return sourcediff --git a/openviking/server/routers/resources.py b/openviking/server/routers/resources.pyindex 45ac3f3db..1f03b5190 100644--- a/openviking/server/routers/resources.py+++ b/openviking/server/routers/resources.py@@ -214,6 +214,7 @@ async def add_resource(wait=request.wait,timeout=request.timeout,allow_local_path_resolution=allow_local_path_resolution,+ enforce_public_remote_targets=True,**kwargs,),)diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex 522493f23..0447a784a 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -11,6 +11,10 @@ import timefrom typing import TYPE_CHECKING, Any, Dict, List, Optionalfrom openviking.server.identity import RequestContext+from openviking.server.local_input_guard import (+ is_remote_resource_source,+ require_remote_resource_source,+)from openviking.storage import VikingDBManagerfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.viking_fs import VikingFS@@ -22,6 +26,7 @@ from openviking.telemetry.resource_summary import (register_wait_telemetry,unregister_wait_telemetry,)+from openviking.utils.network_guard import ensure_public_remote_targetfrom openviking.utils.resource_processor import ResourceProcessorfrom openviking.utils.skill_processor import SkillProcessorfrom openviking_cli.exceptions import (@@ -111,6 +116,7 @@ class ResourceService:watch_interval: float = 0,skip_watch_management: bool = False,allow_local_path_resolution: bool = True,+ enforce_public_remote_targets: bool = False,**kwargs,) -> Dict[str, Any]:"""Add resource to OpenViking (only supports resources scope).@@ -138,6 +144,8 @@ class ResourceService:creating a new one.skip_watch_management: If True, skip watch task management (used by scheduler toavoid recursive watch task creation during scheduled execution)+ enforce_public_remote_targets: When True, reject non-public remote hosts and+ validate each outbound HTTP request URL during fetch.**kwargs: Extra options forwarded to the parser chainReturns:@@ -182,6 +190,9 @@ class ResourceService:raise InvalidArgumentError("watch_interval > 0 requires 'to' to be specified (target URI to watch)")+ if enforce_public_remote_targets and is_remote_resource_source(path):+ path = require_remote_resource_source(path)+ kwargs.setdefault("request_validator", ensure_public_remote_target)result = await self._resource_processor.process_resource(path=path,diff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.pyindex 7fa9e7cef..1fb13b8ce 100644--- a/openviking/utils/media_processor.py+++ b/openviking/utils/media_processor.py@@ -105,19 +105,19 @@ class UnifiedResourceProcessor:"FeishuParser not available. ""Install lark-oapi: pip install 'openviking[bot-feishu]'")- return await parser.parse(url, instruction=instruction)+ return await parser.parse(url, instruction=instruction, **kwargs)# Route git protocols and repo URLs to CodeRepositoryParserif url.startswith(("git@", "git://", "ssh://")) or is_git_repo_url(url):from openviking.parse.parsers.code.code import CodeRepositoryParserparser = CodeRepositoryParser()- return await parser.parse(url, instruction=instruction)+ return await parser.parse(url, instruction=instruction, **kwargs)from openviking.parse.parsers.html import HTMLParserparser = HTMLParser()- return await parser.parse(url, instruction=instruction)+ return await parser.parse(url, instruction=instruction, **kwargs)@staticmethoddef _is_feishu_url(url: str) -> bool:diff --git a/openviking/utils/network_guard.py b/openviking/utils/network_guard.pynew file mode 100644index 000000000..a9dad2c68--- /dev/null+++ b/openviking/utils/network_guard.py@@ -0,0 +1,102 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Network target validation helpers for server-side remote fetches."""++from __future__ import annotations++import ipaddress+import socket+from collections.abc import Callable+from typing import Optional+from urllib.parse import urlparse++from openviking_cli.exceptions import PermissionDeniedError++RequestValidator = Callable[[str], None]++_LOCAL_HOSTNAMES = {+ "localhost",+ "localhost.localdomain",+}+++def extract_remote_host(source: str) -> Optional[str]:+ """Extract the destination host from a remote resource source."""+ if source.startswith("git@"):+ rest = source[4:]+ if ":" not in rest:+ return None+ return rest.split(":", 1)[0].strip().strip("[]")++ parsed = urlparse(source)+ if parsed.hostname is None:+ return None+ return parsed.hostname.strip().strip("[]")+++def _normalize_host(host: str) -> str:+ return host.rstrip(".").lower()+++def _resolve_host_addresses(host: str) -> set[str]:+ try:+ infos = socket.getaddrinfo(host, None, type=socket.SOCK_STREAM)+ except (socket.gaierror, UnicodeError, OSError):+ return set()++ addresses: set[str] = set()+ for family, _, _, _, sockaddr in infos:+ if family not in {socket.AF_INET, socket.AF_INET6}:+ continue+ addr = sockaddr[0]+ if "%" in addr:+ addr = addr.split("%", 1)[0]+ addresses.add(addr)+ return addresses+++def _is_public_ip(address: str) -> bool:+ try:+ return ipaddress.ip_address(address).is_global+ except ValueError:+ return False+++def ensure_public_remote_target(source: str) -> None:+ """Reject loopback, link-local, private, and other non-public targets."""+ host = extract_remote_host(source)+ if not host:+ raise PermissionDeniedError(+ "HTTP server only accepts remote resource URLs with a valid destination host."+ )++ normalized_host = _normalize_host(host)+ if normalized_host in _LOCAL_HOSTNAMES or normalized_host.endswith(".localhost"):+ raise PermissionDeniedError(+ "HTTP server only accepts public remote resource targets; "+ "loopback, link-local, private, and otherwise non-public destinations are not allowed."+ )++ resolved_addresses = _resolve_host_addresses(host)+ if not resolved_addresses:+ return++ non_public = sorted(addr for addr in resolved_addresses if not _is_public_ip(addr))+ if non_public:+ raise PermissionDeniedError(+ "HTTP server only accepts public remote resource targets; "+ f"host '{host}' resolves to non-public address '{non_public[0]}'."+ )+++def build_httpx_request_validation_hooks(+ request_validator: Optional[RequestValidator],+) -> Optional[dict[str, list[Callable]]]:+ """Build httpx request hooks that validate every outbound request URL."""+ if request_validator is None:+ return None++ async def _validate_request(request) -> None:+ request_validator(str(request.url))++ return {"request": [_validate_request]}diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.pyindex 956250ea6..01aaf75df 100644--- a/openviking/utils/resource_processor.py+++ b/openviking/utils/resource_processor.py@@ -18,6 +18,7 @@ from openviking.storage.viking_fs import get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.utils.embedding_utils import index_resourcefrom openviking.utils.summarizer import Summarizer+from openviking_cli.exceptions import OpenVikingErrorfrom openviking_cli.utils import get_loggerfrom openviking_cli.utils.storage import StoragePath@@ -161,6 +162,8 @@ class ResourceProcessor:)telemetry.set("resource.parse.warnings_count", len(parse_result.warnings or []))+ except OpenVikingError:+ raiseexcept Exception as e:result["status"] = "error"result["errors"].append(f"Parse error: {e}")diff --git a/tests/server/test_api_local_input_security.py b/tests/server/test_api_local_input_security.pyindex 8d7ec7c10..351f2be39 100644--- a/tests/server/test_api_local_input_security.py+++ b/tests/server/test_api_local_input_security.py@@ -4,9 +4,16 @@"""Security tests for HTTP server local input handling."""import io+import threadingimport zipfile+from http.server import BaseHTTPRequestHandler, HTTPServerimport httpx+import pytest++from openviking.parse.parsers.html import HTMLParser, URLTypeDetector+from openviking.utils.network_guard import ensure_public_remote_target+from openviking_cli.exceptions import PermissionDeniedErrorasync def test_add_skill_accepts_temp_uploaded_file(@@ -80,6 +87,40 @@ def _build_ovpack_bytes() -> bytes:return buffer.getvalue()+def loopback_http_url():+ body = b"<html><body>loopback secret</body></html>"++ class Handler(BaseHTTPRequestHandler):+ def _write_headers(self) -> None:+ self.send_response(200)+ self.send_header("Content-Type", "text/html; charset=utf-8")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()++ def do_HEAD(self):+ self._write_headers()++ def do_GET(self):+ self._write_headers()+ self.wfile.write(body)++ def log_message(self, format, *args):+ return++ server = HTTPServer(("127.0.0.1", 0), Handler)+ thread = threading.Thread(target=server.serve_forever, daemon=True)+ thread.start()++ try:+ host, port = server.server_address+ yield f"http://{host}:{port}/"+ finally:+ server.shutdown()+ server.server_close()+ thread.join(timeout=3)++async def test_import_ovpack_accepts_temp_uploaded_file(client: httpx.AsyncClient,upload_temp_dir,@@ -152,3 +193,47 @@ async def test_add_resource_rejects_legacy_temp_path_field(client: httpx.AsyncCljson={"temp_path": "upload_resource.md", "reason": "legacy field"},)assert resp.status_code == 422+++async def test_add_resource_rejects_loopback_remote_url(client: httpx.AsyncClient):+ resp = await client.post(+ "/api/v1/resources",+ json={"path": "http://127.0.0.1:8765/", "reason": "ssrf probe"},+ )+ assert resp.status_code == 403+ body = resp.json()+ assert body["status"] == "error"+ assert body["error"]["code"] == "PERMISSION_DENIED"+ assert "public remote resource targets" in body["error"]["message"]+++async def test_add_resource_rejects_private_git_ssh_url(client: httpx.AsyncClient):+ resp = await client.post(+ "/api/v1/resources",+ json={"path": "[email protected]:org/repo.git", "reason": "internal git"},+ )+ assert resp.status_code == 403+ body = resp.json()+ assert body["status"] == "error"+ assert body["error"]["code"] == "PERMISSION_DENIED"+++async def test_url_detector_request_validator_blocks_loopback_head(loopback_http_url: str):+ detector = URLTypeDetector()++ with pytest.raises(PermissionDeniedError):+ await detector.detect(+ loopback_http_url,+ timeout=2.0,+ request_validator=ensure_public_remote_target,+ )+++async def test_html_parser_request_validator_blocks_loopback_fetch(loopback_http_url: str):+ parser = HTMLParser(timeout=2.0)++ with pytest.raises(PermissionDeniedError):+ await parser._fetch_html(+ loopback_http_url,+ request_validator=ensure_public_remote_target,+ )