Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(parser): add network adapter extension point (#6302)

Jiahui Zhoud568563f8314
diff --git a/openviking/parse/__init__.py b/openviking/parse/__init__.py
index f2a1231ab..f0b52d21b 100644
--- a/openviking/parse/__init__.py
+++ b/openviking/parse/__init__.py
@@ -1,43 +1,82 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""Document parsers for various formats."""
-
-from openviking.core.building_tree import BuildingTree
-from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result
-from openviking.parse.directory_scan import (
- CLASS_PROCESSABLE,
- CLASS_UNSUPPORTED,
- ClassifiedFile,
- DirectoryScanResult,
- scan_directory,
-)
-from openviking.parse.mode import ParseMode, ParseModeInput
-from openviking.parse.parsers.base_parser import BaseParser
-from openviking.parse.parsers.html import HTMLParser
-from openviking.parse.parsers.markdown import MarkdownParser
-from openviking.parse.parsers.pdf import PDFParser
-from openviking.parse.parsers.text import TextParser
-from openviking.parse.tree_builder import TreeBuilder
-from openviking.parse.vlm import VLMProcessor
+"""Document parsers and their public lazy exports."""
+
+from __future__ import annotations
+
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from openviking.core.building_tree import BuildingTree
+ from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result
+ from openviking.parse.directory_scan import (
+ CLASS_PROCESSABLE,
+ CLASS_UNSUPPORTED,
+ ClassifiedFile,
+ DirectoryScanResult,
+ scan_directory,
+ )
+ from openviking.parse.mode import ParseMode, ParseModeInput
+ from openviking.parse.parsers.base_parser import BaseParser
+ from openviking.parse.parsers.html import HTMLParser
+ from openviking.parse.parsers.markdown import MarkdownParser
+ from openviking.parse.parsers.pdf import PDFParser
+ from openviking.parse.parsers.text import TextParser
+ from openviking.parse.tree_builder import TreeBuilder
+ from openviking.parse.vlm import VLMProcessor
+
+_EXPORTS = {
+ "BaseParser": ("openviking.parse.parsers.base_parser", "BaseParser"),
+ "ResourceNode": ("openviking.parse.base", "ResourceNode"),
+ "NodeType": ("openviking.parse.base", "NodeType"),
+ "ParseResult": ("openviking.parse.base", "ParseResult"),
+ "create_parse_result": ("openviking.parse.base", "create_parse_result"),
+ "TextParser": ("openviking.parse.parsers.text", "TextParser"),
+ "MarkdownParser": ("openviking.parse.parsers.markdown", "MarkdownParser"),
+ "PDFParser": ("openviking.parse.parsers.pdf", "PDFParser"),
+ "HTMLParser": ("openviking.parse.parsers.html", "HTMLParser"),
+ "TreeBuilder": ("openviking.parse.tree_builder", "TreeBuilder"),
+ "BuildingTree": ("openviking.core.building_tree", "BuildingTree"),
+ "VLMProcessor": ("openviking.parse.vlm", "VLMProcessor"),
+ "CLASS_PROCESSABLE": ("openviking.parse.directory_scan", "CLASS_PROCESSABLE"),
+ "CLASS_UNSUPPORTED": ("openviking.parse.directory_scan", "CLASS_UNSUPPORTED"),
+ "ClassifiedFile": ("openviking.parse.directory_scan", "ClassifiedFile"),
+ "DirectoryScanResult": ("openviking.parse.directory_scan", "DirectoryScanResult"),
+ "scan_directory": ("openviking.parse.directory_scan", "scan_directory"),
+ "ParseMode": ("openviking.parse.mode", "ParseMode"),
+ "ParseModeInput": ("openviking.parse.mode", "ParseModeInput"),
+}
+
+
+def __getattr__(name: str) -> Any:
+ try:
+ module_name, attr_name = _EXPORTS[name]
+ except KeyError as exc:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from exc
+
+ value = getattr(import_module(module_name), attr_name)
+ globals()[name] = value
+ return value
+
+
+def __dir__() -> list[str]:
+ return sorted(list(globals().keys()) + list(__all__))
+
__all__ = [
- # Base classes and helpers
"BaseParser",
"ResourceNode",
"NodeType",
"ParseResult",
"create_parse_result",
- # Document parsers (core)
"TextParser",
"MarkdownParser",
"PDFParser",
"HTMLParser",
- # Tree builder
"TreeBuilder",
"BuildingTree",
- # VLM
"VLMProcessor",
- # Directory scan (phase-one validation)
"CLASS_PROCESSABLE",
"CLASS_UNSUPPORTED",
"ClassifiedFile",
diff --git a/openviking/parse/accessors/git_accessor.py b/openviking/parse/accessors/git_accessor.py
index dad21ac81..cf46230ca 100644
--- a/openviking/parse/accessors/git_accessor.py
+++ b/openviking/parse/accessors/git_accessor.py
@@ -19,6 +19,13 @@ from pathlib import Path, PurePosixPath
from typing import Optional, Tuple, Union
from urllib.parse import quote, urlparse
+from openviking.parse.network import (
+ ParserNetworkContext,
+ ParserNetworkOperation,
+ open_parser_url,
+ overlay_parser_subprocess_env,
+)
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.utils import (
is_github_url,
is_gitlab_url,
@@ -35,7 +42,6 @@ from openviking.utils.git_auth import (
parse_git_http_auth_config,
raise_git_auth_error,
)
-from openviking.parse.temp import get_parser_download_temp_dir
from openviking_cli.utils.logger import get_logger
from .base import DataAccessor, LocalResource, SourceType
@@ -390,6 +396,10 @@ class GitAccessor(DataAccessor):
"""Clone a git repository into target_dir; return the repo name."""
name = self._get_repo_name(url)
logger.info(f"[GitAccessor] Cloning {url} to {target_dir}...")
+ network_env = overlay_parser_subprocess_env(
+ ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, url),
+ env,
+ )
clone_args = [
"git",
@@ -401,19 +411,19 @@ class GitAccessor(DataAccessor):
if branch and not commit:
clone_args.extend(["--branch", branch])
clone_args.extend([url, target_dir])
- await self._run_git(clone_args, env=env)
+ await self._run_git(clone_args, env=network_env)
if commit:
try:
await self._run_git(
["git", "-C", target_dir, "fetch", "origin", commit],
- env=env,
+ env=network_env,
)
except RuntimeError:
try:
await self._run_git(
["git", "-C", target_dir, "fetch", "--all", "--tags", "--prune"],
- env=env,
+ env=network_env,
)
except RuntimeError:
pass
@@ -422,7 +432,7 @@ class GitAccessor(DataAccessor):
try:
await self._run_git(
["git", "-C", target_dir, "fetch", "--unshallow", "origin"],
- env=env,
+ env=network_env,
)
except RuntimeError:
pass
@@ -437,7 +447,7 @@ class GitAccessor(DataAccessor):
"origin",
"+refs/heads/*:refs/remotes/origin/*",
],
- env=env,
+ env=network_env,
)
ok = await self._has_commit(target_dir, commit, env=env)
if not ok:
@@ -493,7 +503,8 @@ class GitAccessor(DataAccessor):
headers["Authorization"] = f"token {github_token}"
req = urllib.request.Request(zip_url, headers=headers)
- with urllib.request.urlopen(req, timeout=1800) as resp, open(zip_path, "wb") as f:
+ context = ParserNetworkContext(ParserNetworkOperation.GIT_ARCHIVE, zip_url)
+ with open_parser_url(context, req, timeout=1800) as resp, open(zip_path, "wb") as f:
shutil.copyfileobj(resp, f)
try:
@@ -586,7 +597,8 @@ class GitAccessor(DataAccessor):
headers = {"User-Agent": "OpenViking"}
req = urllib.request.Request(zip_url, headers=headers)
- with urllib.request.urlopen(req, timeout=1800) as resp, open(zip_path, "wb") as f:
+ context = ParserNetworkContext(ParserNetworkOperation.GIT_ARCHIVE, zip_url)
+ with open_parser_url(context, req, timeout=1800) as resp, open(zip_path, "wb") as f:
shutil.copyfileobj(resp, f)
try:
diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.py
index d94ba2034..22b9fe2fa 100644
--- a/openviking/parse/accessors/http_accessor.py
+++ b/openviking/parse/accessors/http_accessor.py
@@ -24,13 +24,18 @@ from typing import Any, Dict, Mapping, Optional, Tuple, Union
from urllib.parse import unquote, urlparse
from openviking.parse.base import lazy_import
-from openviking.parse.temp import get_parser_download_temp_dir
+from openviking.parse.network import (
+ ParserNetworkContext,
+ ParserNetworkOperation,
+ create_parser_httpx_client,
+)
from openviking.parse.parsers.constants import CODE_EXTENSIONS
from openviking.parse.parsers.media.constants import (
AUDIO_EXTENSIONS,
IMAGE_EXTENSIONS,
VIDEO_EXTENSIONS,
)
+from openviking.parse.temp import get_parser_download_temp_dir
from openviking.utils import is_code_hosting_blob_url
from openviking.utils.exceptions import error_code_from_http_status
from openviking.utils.network_guard import build_httpx_request_validation_hooks
@@ -215,8 +220,13 @@ class URLTypeDetector:
client_kwargs["event_hooks"] = event_hooks
client_kwargs["trust_env"] = False
- async with httpx.AsyncClient(**client_kwargs) as client:
- response = await client.head(url, headers=headers)
+ client = create_parser_httpx_client(
+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),
+ client_cls=httpx.AsyncClient,
+ **client_kwargs,
+ )
+ async with client as active_client:
+ response = await active_client.head(url, headers=headers)
meta["status_code"] = response.status_code
if not (200 <= response.status_code < 300):
@@ -584,10 +594,15 @@ class HTTPAccessor(DataAccessor):
client_kwargs["event_hooks"] = event_hooks
client_kwargs["trust_env"] = False
- async with httpx.AsyncClient(**client_kwargs) as client:
+ client = create_parser_httpx_client(
+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),
+ client_cls=httpx.AsyncClient,
+ **client_kwargs,
+ )
+ async with client as active_client:
headers = self._request_headers(tos_signature, tos_access)
try:
- response = await client.get(url, headers=headers)
+ response = await active_client.get(url, headers=headers)
response.raise_for_status()
except httpx.ConnectError as e:
cause: BaseException | None = e
diff --git a/openviking/parse/accessors/web_crawler/middlewares.py b/openviking/parse/accessors/web_crawler/middlewares.py
index b9ae22998..c01b092b7 100644
--- a/openviking/parse/accessors/web_crawler/middlewares.py
+++ b/openviking/parse/accessors/web_crawler/middlewares.py
@@ -2,8 +2,15 @@
# SPDX-License-Identifier: AGPL-3.0
"""Scrapy downloader middlewares for the recursive web crawler."""
+import ipaddress
+from collections.abc import Mapping
+from urllib.parse import urlparse
+from urllib.request import proxy_bypass_environment
+
from scrapy.exceptions import IgnoreRequest
+_PARSER_PROXY_MANAGED = "_openviking_parser_proxy_managed"
+
class RequestValidatorMiddleware:
"""Pre-flight SSRF guard: validate every outbound URL, including redirects/retries."""
@@ -17,3 +24,59 @@ class RequestValidatorMiddleware:
except Exception as exc:
raise IgnoreRequest(f"Blocked by request_validator: {exc}") from exc
return None
+
+
+def _proxy_value(environment: Mapping[str, str], scheme: str) -> str | None:
+ return environment.get(f"{scheme}_proxy") or environment.get(f"{scheme.upper()}_PROXY")
+
+
+def _bypasses_proxy(environment: Mapping[str, str], host: str, port: int | None) -> bool:
+ no_proxy = _proxy_value(environment, "no")
+ if not no_proxy:
+ return False
+ host_with_port = f"{host}:{port}" if port is not None else host
+ if proxy_bypass_environment(host_with_port, {"no": no_proxy}):
+ return True
+ try:
+ address = ipaddress.ip_address(host)
+ except ValueError:
+ return False
+ for value in no_proxy.split(","):
+ try:
+ if address in ipaddress.ip_network(value.strip(), strict=False):
+ return True
+ except ValueError:
+ continue
+ return False
+
+
+class ParserNetworkProxyMiddleware:
+ """Apply the parser adapter's child-only proxy policy to each Scrapy request."""
+
+ def process_request(self, request, spider):
+ # Scrapy copies request.meta when following redirects. Keep a marker
+ # for requests whose proxy was selected here so a redirect can choose
+ # the proxy for its *new* scheme instead of reusing the old one. A
+ # caller-supplied proxy remains authoritative and is never rewritten.
+ managed = bool(request.meta.get(_PARSER_PROXY_MANAGED))
+ if "proxy" in request.meta and not managed:
+ return None
+ environment = getattr(spider, "network_env", None) or {}
+ parsed = urlparse(request.url)
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:
+ return None
+ if not any(_proxy_value(environment, scheme) for scheme in ("http", "https", "all")):
+ # No parser proxy was supplied. Leave Scrapy's normal
+ # HttpProxyMiddleware behavior unchanged for deployments that do
+ # not install the adapter.
+ return None
+ request.meta[_PARSER_PROXY_MANAGED] = True
+ if _bypasses_proxy(environment, parsed.hostname, parsed.port):
+ # ``HttpProxyMiddleware`` runs later and otherwise sees the child
+ # proxy environment and installs a proxy again (notably for CIDR
+ # NO_PROXY entries). A None proxy is its documented opt-out.
+ request.meta["proxy"] = None
+ return None
+ proxy = _proxy_value(environment, parsed.scheme) or _proxy_value(environment, "all")
+ request.meta["proxy"] = proxy
+ return None
diff --git a/openviking/parse/accessors/web_crawler/scrapy_spider.py b/openviking/parse/accessors/web_crawler/scrapy_spider.py
index 9cf344242..39deccefd 100644
--- a/openviking/parse/accessors/web_crawler/scrapy_spider.py
+++ b/openviking/parse/accessors/web_crawler/scrapy_spider.py
@@ -4,6 +4,7 @@
import os
from collections.abc import Iterator
+from typing import Mapping
from urllib.parse import urljoin, urlparse
import scrapy
@@ -14,7 +15,6 @@ from openviking.parse.accessors.http_accessor import URLType, URLTypeDetector
from openviking.parse.accessors.web_crawler.config import CrawlConfig
from openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage
-
_DOWNLOAD_URL_TYPES = frozenset(
{
URLType.DOWNLOAD_PDF,
@@ -64,6 +64,7 @@ class OpenVikingWebSpider(scrapy.Spider):
config: CrawlConfig,
collector: list[CrawledPage],
download_collector: list[CrawledDownload],
+ network_env: Mapping[str, str] | None = None,
**kwargs,
):
super().__init__(**kwargs)
@@ -71,6 +72,7 @@ class OpenVikingWebSpider(scrapy.Spider):
self.config = config
self.collector = collector
self.download_collector = download_collector
+ self.network_env = dict(network_env or {})
self.root_host = urlparse(root_url).netloc.lower()
self._success_count = 0
self._seen_download_urls: set[str] = set()
diff --git a/openviking/parse/accessors/web_crawler/web_crawler.py b/openviking/parse/accessors/web_crawler/web_crawler.py
index 05bb2e46e..0e449fa40 100644
--- a/openviking/parse/accessors/web_crawler/web_crawler.py
+++ b/openviking/parse/accessors/web_crawler/web_crawler.py
@@ -4,7 +4,10 @@
import asyncio
import multiprocessing
+import os
import queue
+from contextlib import contextmanager
+from typing import Iterator, Mapping
from openviking.parse.accessors.web_crawler.config import CrawlConfig
from openviking.parse.accessors.web_crawler.models import (
@@ -12,9 +15,45 @@ from openviking.parse.accessors.web_crawler.models import (
CrawledPage,
CrawlResult,
)
+from openviking.parse.network import (
+ ParserNetworkContext,
+ ParserNetworkOperation,
+ overlay_parser_subprocess_env,
+)
+
+
+@contextmanager
+def _temporary_environment(overlay: Mapping[str, str]) -> Iterator[None]:
+ """Install child-only network variables without leaking across worker reuse."""
+ previous = {name: os.environ.get(name) for name in overlay}
+ os.environ.update(overlay)
+ try:
+ yield
+ finally:
+ for name, value in previous.items():
+ if value is None:
+ os.environ.pop(name, None)
+ else:
+ os.environ[name] = value
-def _run_crawl_worker(root_url: str, config: CrawlConfig, result_queue) -> None:
+
+def _run_crawl_worker(
+ root_url: str,
+ config: CrawlConfig,
+ result_queue,
+ network_env: Mapping[str, str] | None = None,
+) -> None:
+ with _temporary_environment(network_env or {}):
+ _run_crawl_worker_with_environment(root_url, config, result_queue, network_env or {})
+
+
+def _run_crawl_worker_with_environment(
+ root_url: str,
+ config: CrawlConfig,
+ result_queue,
+ network_env: Mapping[str, str],
+) -> None:
from scrapy.utils.reactor import install_reactor
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
@@ -33,6 +72,7 @@ def _run_crawl_worker(root_url: str, config: CrawlConfig, result_queue) -> None:
config=config,
collector=pages,
download_collector=downloads,
+ network_env=network_env,
)
try:
process.start()
@@ -67,8 +107,9 @@ def _build_settings(config: CrawlConfig):
"TELNETCONSOLE_ENABLED": False,
"USER_AGENT": "OpenViking/0.4 (+recursive-web-crawler)",
"DOWNLOADER_MIDDLEWARES": {
- "openviking.parse.accessors.web_crawler.middlewares."
- "RequestValidatorMiddleware": 50,
+ "openviking.parse.accessors.web_crawler.middlewares.RequestValidatorMiddleware": 50,
+ "openviking.parse.accessors.web_crawler.middlewares.ParserNetworkProxyMiddleware": 100,
+ "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
},
}
)
@@ -81,9 +122,13 @@ class ScrapyWebCrawler:
async def crawl(self, root_url: str) -> CrawlResult:
context = multiprocessing.get_context("spawn")
result_queue = context.Queue()
+ network_env = overlay_parser_subprocess_env(
+ ParserNetworkContext(ParserNetworkOperation.WEB_CRAWL, root_url),
+ {},
+ )
process = context.Process(
target=_run_crawl_worker,
- args=(root_url, self.config, result_queue),
+ args=(root_url, self.config, result_queue, network_env),
)
process.start()
try:
diff --git a/openviking/parse/accessors/web_feed_accessor.py b/openviking/parse/accessors/web_feed_accessor.py
index c9c9cc21c..d289e9265 100644
--- a/openviking/parse/accessors/web_feed_accessor.py
+++ b/openviking/parse/accessors/web_feed_accessor.py
@@ -37,6 +37,11 @@ from urllib.parse import unquote, urljoin, urlparse
from urllib.robotparser import RobotFileParser
from openviking.parse.base import lazy_import
+from openviking.parse.network import (
+ ParserNetworkContext,
+ ParserNetworkOperation,
+ create_parser_httpx_client,
+)
from openviking.parse.temp import get_parser_download_temp_dir
from openviking.utils.network_guard import build_httpx_request_validation_hooks
from openviking_cli.utils.logger import get_logger
@@ -297,19 +302,24 @@ class WebFeedAccessor(DataAccessor):
try:
client_kwargs = _build_httpx_client_kwargs(request_validator, cfg["request_timeout"])
httpx = lazy_import("httpx")
- async with httpx.AsyncClient(**client_kwargs) as client:
+ client = create_parser_httpx_client(
+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, source_url),
+ client_cls=httpx.AsyncClient,
+ **client_kwargs,
+ )
+ async with client as active_client:
# The given URL may already be a sitemap/feed, or it may be a plain
# page / bare domain (e.g. args={"site": True} on "https://t0saki.com").
# In the latter case, auto-discover the site's sitemap/RSS.
- feed_url, prefetched = await self._resolve_feed_source(source_url, client)
+ feed_url, prefetched = await self._resolve_feed_source(source_url, active_client)
entries, feed_kind = await self._collect(
- feed_url, client, cfg, prefetched=prefetched
+ feed_url, active_client, cfg, prefetched=prefetched
)
if not entries:
raise RuntimeError(f"No pages found in sitemap/feed: {feed_url}")
entries = self._filter_entries(entries, feed_url, cfg, include, exclude)
- entries = await self._apply_robots(entries, feed_url, client, cfg)
+ entries = await self._apply_robots(entries, feed_url, active_client, cfg)
if len(entries) > cfg["max_pages"]:
logger.warning(
@@ -323,7 +333,7 @@ class WebFeedAccessor(DataAccessor):
if not entries:
raise RuntimeError(f"No ingestible pages remained after filtering: {feed_url}")
- pages = await self._mirror(entries, client, temp_dir, cfg)
+ pages = await self._mirror(entries, active_client, temp_dir, cfg)
if not pages:
raise RuntimeError(f"Failed to fetch any page from: {feed_url}")
@@ -793,11 +803,16 @@ async def _discover_feed_hint(url: str, request_validator) -> Optional[str]:
httpx = lazy_import("httpx")
client_kwargs = _build_httpx_client_kwargs(request_validator, 5.0)
- async with httpx.AsyncClient(**client_kwargs) as client:
+ client = create_parser_httpx_client(
+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),
+ client_cls=httpx.AsyncClient,
+ **client_kwargs,
+ )
+ async with client as active_client:
# Fetch the page once for <link> autodiscovery (best-effort).
page_content = b""
try:
- response = await client.get(url, headers={"User-Agent": _USER_AGENT})
+ response = await active_client.get(url, headers={"User-Agent": _USER_AGENT})
if (
response.status_code < 400
and "html" in response.headers.get("content-type", "").lower()
@@ -806,9 +821,9 @@ async def _discover_feed_hint(url: str, request_validator) -> Optional[str]:
except Exception:
pass
- for candidate in await _gather_feed_candidates(client, url, page_content):
+ for candidate in await _gather_feed_candidates(active_client, url, page_content):
try:
- response = await client.get(candidate, headers={"User-Agent": _USER_AGENT})
+ response = await active_client.get(candidate, headers={"User-Agent": _USER_AGENT})
if response.status_code >= 400:
continue
info = _classify_and_count(response.content)
diff --git a/openviking/parse/network.py b/openviking/parse/network.py
new file mode 100644
index 000000000..cd31483b4
--- /dev/null
+++ b/openviking/parse/network.py
@@ -0,0 +1,163 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Stable network extension point for parser resource acquisition."""
+
+from __future__ import annotations
+
+import os
+import urllib.request
+from dataclasses import dataclass
+from enum import Enum
+from threading import RLock
+from typing import Any, Mapping, Protocol
+
+import httpx
+
+
+class ParserNetworkOperation(str, Enum):
+ """Network operations whose routing may be extended by a distribution."""
+
+ HTTP_RESOURCE = "http_resource"
+ GIT_ARCHIVE = "git_archive"
+ GIT_COMMAND = "git_command"
+ WEB_CRAWL = "web_crawl"
+
+
+@dataclass(frozen=True)
+class ParserNetworkContext:
+ """Strongly typed description of one parser network boundary."""
+
+ operation: ParserNetworkOperation
+ source_url: str
+
+ def __post_init__(self) -> None:
+ if not isinstance(self.operation, ParserNetworkOperation):
+ raise TypeError("operation must be a ParserNetworkOperation")
+ if not isinstance(self.source_url, str) or not self.source_url.strip():
+ raise ValueError("source_url must be a non-empty string")
+
+
+class ParserNetworkAdapter(Protocol):
+ """Distribution-owned routing hooks for parser network operations."""
+
+ def supports(self, context: ParserNetworkContext) -> bool: ...
+
+ def create_httpx_transport(
+ self,
+ context: ParserNetworkContext,
+ ) -> httpx.AsyncBaseTransport | None: ...
+
+ def create_urllib_opener(
+ self,
+ context: ParserNetworkContext,
+ ) -> urllib.request.OpenerDirector | None: ...
+
+ def subprocess_env_overlay(
+ self,
+ context: ParserNetworkContext,
+ ) -> Mapping[str, str]: ...
+
+
+_adapter: ParserNetworkAdapter | None = None
+_lock = RLock()
+
+
+def register_parser_network_adapter(adapter: ParserNetworkAdapter) -> None:
+ """Register one process-wide parser network adapter.
+
+ Re-registering the same object is idempotent. Replacing it is rejected so
+ unrelated extensions cannot silently take ownership of parser egress.
+ """
+
+ global _adapter
+ with _lock:
+ if _adapter is adapter:
+ return
+ if _adapter is not None:
+ raise RuntimeError("A parser network adapter is already registered")
+ _adapter = adapter
+
+
+def reset_parser_network_adapter() -> None:
+ """Reset the adapter. Intended for tests and runtime reinitialization."""
+
+ global _adapter
+ with _lock:
+ _adapter = None
+
+
+def _adapter_for(context: ParserNetworkContext) -> ParserNetworkAdapter | None:
+ with _lock:
+ adapter = _adapter
+ if adapter is not None and adapter.supports(context):
+ return adapter
+ return None
+
+
+def create_parser_httpx_client(
+ context: ParserNetworkContext,
+ *,
+ client_cls: type[httpx.AsyncClient] = httpx.AsyncClient,
+ **client_kwargs: Any,
+) -> httpx.AsyncClient:
+ """Create a parser HTTP client while preserving caller-owned safeguards."""
+
+ adapter = _adapter_for(context)
+ if adapter is not None:
+ transport = adapter.create_httpx_transport(context)
+ if transport is not None:
+ if client_kwargs.get("transport") is not None:
+ raise RuntimeError("Parser network adapter cannot replace an explicit transport")
+ client_kwargs["transport"] = transport
+ # The adapter transport owns proxy/direct routing. Environment
+ # lookup here would reintroduce process-wide proxy side effects.
+ client_kwargs["trust_env"] = False
+ return client_cls(**client_kwargs)
+
+
+def open_parser_url(
+ context: ParserNetworkContext,
+ request: urllib.request.Request | str,
+ **open_kwargs: Any,
+):
+ """Open one parser URL through an adapter opener when available."""
+
+ adapter = _adapter_for(context)
+ if adapter is not None:
+ opener = adapter.create_urllib_opener(context)
+ if opener is not None:
+ return opener.open(request, **open_kwargs)
+ return urllib.request.urlopen(request, **open_kwargs)
+
+
+def overlay_parser_subprocess_env(
+ context: ParserNetworkContext,
+ base_env: Mapping[str, str] | None,
+) -> dict[str, str] | None:
+ """Merge adapter variables into a child environment without mutating it."""
+
+ adapter = _adapter_for(context)
+ if adapter is None:
+ return base_env if isinstance(base_env, dict) or base_env is None else dict(base_env)
+
+ overlay = dict(adapter.subprocess_env_overlay(context))
+ if not overlay:
+ return base_env if isinstance(base_env, dict) or base_env is None else dict(base_env)
+ if not all(isinstance(key, str) and isinstance(value, str) for key, value in overlay.items()):
+ raise TypeError("Parser subprocess environment overlay must contain string pairs")
+
+ merged = dict(os.environ if base_env is None else base_env)
+ merged.update(overlay)
+ return merged
+
+
+__all__ = [
+ "ParserNetworkAdapter",
+ "ParserNetworkContext",
+ "ParserNetworkOperation",
+ "create_parser_httpx_client",
+ "open_parser_url",
+ "overlay_parser_subprocess_env",
+ "register_parser_network_adapter",
+ "reset_parser_network_adapter",
+]
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index df8a3c1c1..a607fe7fc 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -30,6 +30,11 @@ from openviking.core.namespace import is_content_root_uri
from openviking.observability.http_error_context import sanitize_public_http_error
from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.mode import ParseMode, normalize_parse_mode
+from openviking.parse.network import (
+ ParserNetworkContext,
+ ParserNetworkOperation,
+ overlay_parser_subprocess_env,
+)
from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS
from openviking.resource.feishu_watch_auth import (
FEISHU_ACCESS_TOKEN_ARG,
@@ -1488,6 +1493,10 @@ class ResourceService:
proc = None
try:
env = build_git_http_auth_env(auth_config, source) if auth_config is not None else None
+ env = overlay_parser_subprocess_env(
+ ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, source),
+ env,
+ )
proc = await asyncio.create_subprocess_exec(
"git",
"ls-remote",
diff --git a/tests/misc/test_network_guard.py b/tests/misc/test_network_guard.py
index ac7cd2be3..8be5a8bd6 100644
--- a/tests/misc/test_network_guard.py
+++ b/tests/misc/test_network_guard.py
@@ -5,10 +5,18 @@
from __future__ import annotations
from types import SimpleNamespace
-from unittest.mock import AsyncMock, patch
+from unittest.mock import patch
+import httpx
import pytest
+from openviking.parse.network import (
+ ParserNetworkContext,
+ ParserNetworkOperation,
+ create_parser_httpx_client,
+ register_parser_network_adapter,
+ reset_parser_network_adapter,
+)
from openviking.utils.network_guard import (
_get_allowed_code_hosting_domains,
_is_public_ip,
@@ -287,48 +295,43 @@ class TestEnsurePublicRemoteTarget:
class TestBuildHttpxRequestValidationHooks:
- """Verify httpx hook construction."""
+ """Verify optional validation and enforcement during parser downloads."""
def test_returns_none_when_no_validator(self) -> None:
assert build_httpx_request_validation_hooks(None) is None
- def test_returns_request_hook_dict(self) -> None:
- def dummy_validator(url: str) -> None:
- pass
-
- hooks = build_httpx_request_validation_hooks(dummy_validator)
- assert hooks is not None
- assert "request" in hooks
- assert len(hooks["request"]) == 1
-
@pytest.mark.asyncio
- async def test_hook_calls_validator_with_url(self) -> None:
- calls: list[str] = []
-
- def tracking_validator(url: str) -> None:
- calls.append(url)
-
- hooks = build_httpx_request_validation_hooks(tracking_validator)
- assert hooks is not None
-
- mock_request = AsyncMock()
- mock_request.url = "http://example.com/test"
-
- hook_fn = hooks["request"][0]
- await hook_fn(mock_request)
-
- assert calls == ["http://example.com/test"]
-
- @pytest.mark.asyncio
- async def test_hook_propagates_validator_exception(self) -> None:
- def failing_validator(url: str) -> None:
- raise PermissionDeniedError("blocked")
-
- hooks = build_httpx_request_validation_hooks(failing_validator)
- assert hooks is not None
-
- mock_request = AsyncMock()
- mock_request.url = "http://evil.com"
-
- with pytest.raises(PermissionDeniedError, match="blocked"):
- await hooks["request"][0](mock_request)
+ async def test_parser_adapter_preserves_redirect_validation(self) -> None:
+ source = "https://example.com/document.txt"
+ private_target = "http://127.0.0.1/admin"
+ validated = []
+ downloaded = []
+
+ def validate(url):
+ validated.append(url)
+ if url == private_target:
+ raise PermissionDeniedError("blocked")
+
+ async def download(request):
+ downloaded.append(str(request.url))
+ return httpx.Response(302, headers={"Location": private_target})
+
+ register_parser_network_adapter(
+ SimpleNamespace(
+ supports=lambda context: context.operation is ParserNetworkOperation.HTTP_RESOURCE,
+ create_httpx_transport=lambda context: httpx.MockTransport(download),
+ )
+ )
+ try:
+ async with create_parser_httpx_client(
+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, source),
+ follow_redirects=True,
+ event_hooks=build_httpx_request_validation_hooks(validate),
+ ) as client:
+ with pytest.raises(PermissionDeniedError, match="blocked"):
+ await client.get(source)
+ finally:
+ reset_parser_network_adapter()
+
+ assert validated == [source, private_target]
+ assert downloaded == [source]
diff --git a/tests/parse/accessors/web_crawler/test_middlewares.py b/tests/parse/accessors/web_crawler/test_middlewares.py
index efd7d0265..6b6010240 100644
--- a/tests/parse/accessors/web_crawler/test_middlewares.py
+++ b/tests/parse/accessors/web_crawler/test_middlewares.py
@@ -1,53 +1,75 @@
-from unittest.mock import MagicMock
+from types import SimpleNamespace
+from unittest.mock import patch
import pytest
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
+from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
+from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import IgnoreRequest
+from scrapy.http import Response
+from scrapy.settings import Settings
from openviking.parse.accessors.web_crawler.config import CrawlConfig
-from openviking.parse.accessors.web_crawler.middlewares import RequestValidatorMiddleware
+from openviking.parse.accessors.web_crawler.middlewares import (
+ ParserNetworkProxyMiddleware,
+ RequestValidatorMiddleware,
+)
-def _make_spider(validator):
- spider = MagicMock()
- spider.config = CrawlConfig(request_validator=validator)
- return spider
+class TestParserRequestRouting:
+ def test_raises_ignore_when_validator_rejects(self):
+ def reject(url):
+ raise ValueError(f"private address: {url}")
+ spider = SimpleNamespace(config=CrawlConfig(request_validator=reject))
+ request = Request("http://169.254.169.254/latest/meta-data/")
+ with pytest.raises(IgnoreRequest, match="private address"):
+ RequestValidatorMiddleware().process_request(request, spider)
-def _make_request(url):
- request = MagicMock()
- request.url = url
- return request
+ def test_redirects_revalidate_and_select_proxy_or_direct_route(self):
+ environment = {
+ "HTTP_PROXY": "http://http-proxy.example:8118",
+ "HTTPS_PROXY": "http://https-proxy.example:8118",
+ "NO_PROXY": "93.184.0.0/16",
+ }
+ seen = []
+ spider = SimpleNamespace(
+ config=CrawlConfig(request_validator=seen.append),
+ network_env=environment,
+ )
+ validator = RequestValidatorMiddleware()
+ routing = ParserNetworkProxyMiddleware()
+ redirects = RedirectMiddleware.from_crawler(Crawler(Spider, Settings()))
+ with patch.dict("os.environ", environment, clear=True):
+ native_proxy = HttpProxyMiddleware()
+ request = Request("http://example.com/start")
+ validator.process_request(request, spider)
+ routing.process_request(request, spider)
+ native_proxy.process_request(request, spider)
+ assert request.meta["proxy"] == environment["HTTP_PROXY"]
-class TestRequestValidatorMiddleware:
- def test_passes_through_when_validator_accepts(self):
- mw = RequestValidatorMiddleware()
- spider = _make_spider(validator=lambda _url: None)
- request = _make_request("https://example.com/")
- assert mw.process_request(request, spider) is None
+ response = Response(
+ request.url, status=301, headers={"Location": "https://example.com/final"}
+ )
+ request = redirects.process_response(request, response, spider)
+ validator.process_request(request, spider)
+ routing.process_request(request, spider)
+ native_proxy.process_request(request, spider)
+ assert request.meta["proxy"] == environment["HTTPS_PROXY"]
- def test_raises_ignore_when_validator_rejects(self):
- mw = RequestValidatorMiddleware()
-
- def reject(url):
- raise ValueError(f"private address: {url}")
+ response = Response(
+ request.url, status=302, headers={"Location": "http://93.184.216.34/article"}
+ )
+ request = redirects.process_response(request, response, spider)
+ validator.process_request(request, spider)
+ routing.process_request(request, spider)
+ native_proxy.process_request(request, spider)
+ assert request.meta["proxy"] is None
- spider = _make_spider(validator=reject)
- request = _make_request("http://169.254.169.254/latest/meta-data/")
- with pytest.raises(IgnoreRequest) as exc_info:
- mw.process_request(request, spider)
- assert "private address" in str(exc_info.value)
-
- def test_no_op_when_validator_unset(self):
- mw = RequestValidatorMiddleware()
- spider = _make_spider(validator=None)
- request = _make_request("https://example.com/")
- assert mw.process_request(request, spider) is None
-
- def test_validator_receives_request_url(self):
- mw = RequestValidatorMiddleware()
- seen: list[str] = []
- spider = _make_spider(validator=lambda url: seen.append(url))
- request = _make_request("https://example.com/redirected-target")
- mw.process_request(request, spider)
- assert seen == ["https://example.com/redirected-target"]
+ assert seen == [
+ "http://example.com/start",
+ "https://example.com/final",
+ "http://93.184.216.34/article",
+ ]
diff --git a/tests/service/test_resource_service_git_preflight.py b/tests/service/test_resource_service_git_preflight.py
index eb97ded11..da91054e1 100644
--- a/tests/service/test_resource_service_git_preflight.py
+++ b/tests/service/test_resource_service_git_preflight.py
@@ -3,6 +3,7 @@
"""Regression tests for Git preflight process cleanup."""
import asyncio
+import base64
import os
import signal
from types import SimpleNamespace
@@ -11,6 +12,11 @@ from unittest.mock import AsyncMock, Mock, call
import pytest
from openviking.parse.mode import ParseMode
+from openviking.parse.network import (
+ ParserNetworkOperation,
+ register_parser_network_adapter,
+ reset_parser_network_adapter,
+)
from openviking.server.identity import RequestContext, Role
from openviking.service import resource_service as resource_service_module
from openviking.service.resource_service import ResourceService
@@ -20,16 +26,16 @@ from openviking_cli.session.user_id import UserIdentifier
pytestmark = pytest.mark.skipif(os.name != "posix", reason="process groups are POSIX-only")
-async def test_account_github_token_is_used_for_repository_preflight(monkeypatch):
+async def test_account_token_and_parser_proxy_reach_repository_preflight(monkeypatch):
token_resolver = AsyncMock(return_value="account-token")
processor = SimpleNamespace(github_token_for=token_resolver)
service = ResourceService(resource_processor=processor)
- service._preflight_git_source = AsyncMock(
- return_value=SimpleNamespace(
- source_name="private",
- source_path="https://github.com/org/private",
- source_format="repository",
- )
+ process = Mock(returncode=0)
+ process.communicate = AsyncMock(return_value=(b"refs/heads/main\n", b""))
+ execute = AsyncMock(return_value=process)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)
+ monkeypatch.setattr(
+ resource_service_module, "parse_code_hosting_url", lambda _source: "org/private"
)
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
monkeypatch.setattr(resource_service_module, "is_github_url", lambda _path: True)
@@ -38,19 +44,41 @@ async def test_account_github_token_is_used_for_repository_preflight(monkeypatch
role=Role.USER,
)
- plan = await service._prepare_standard_source_plan(
- path="https://github.com/org/private",
- ctx=ctx,
- mode=ParseMode.DEFAULT,
- allow_local_path_resolution=False,
- processor_kwargs={},
- watch_auth_state=None,
+ register_parser_network_adapter(
+ SimpleNamespace(
+ supports=lambda context: context.operation is ParserNetworkOperation.GIT_COMMAND,
+ subprocess_env_overlay=lambda context: {"HTTPS_PROXY": "http://proxy.example:8118"},
+ )
)
+ try:
+ plan = await service._prepare_standard_source_plan(
+ path="https://github.com/org/private",
+ ctx=ctx,
+ mode=ParseMode.DEFAULT,
+ allow_local_path_resolution=False,
+ processor_kwargs={},
+ watch_auth_state=None,
+ )
+ finally:
+ reset_parser_network_adapter()
token_resolver.assert_awaited_once_with("https://github.com/org/private", ctx)
- preflight_auth = service._preflight_git_source.await_args.kwargs["auth_config"]
- assert preflight_auth.username == "oauth2"
- assert preflight_auth.token == "account-token"
+ assert execute.await_args.args == (
+ "git",
+ "ls-remote",
+ "--heads",
+ "https://github.com/org/private",
+ )
+ env = execute.await_args.kwargs["env"]
+ assert env["HTTPS_PROXY"] == "http://proxy.example:8118"
+ entries = {
+ env[f"GIT_CONFIG_KEY_{index}"]: env[f"GIT_CONFIG_VALUE_{index}"]
+ for index in range(int(env["GIT_CONFIG_COUNT"]))
+ }
+ encoded = base64.b64encode(b"oauth2:account-token").decode("ascii")
+ assert entries["http.https://github.com/org/private.extraHeader"] == (
+ f"Authorization: Basic {encoded}"
+ )
assert plan is not None
assert plan.task_auth == {}
assert "account-token" not in str(plan.processor_args)