Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(parser): add network adapter extension point (#6302)
Jiahui Zhoud568563f8314
diff --git a/openviking/parse/__init__.py b/openviking/parse/__init__.pyindex f2a1231ab..f0b52d21b 100644--- a/openviking/parse/__init__.py+++ b/openviking/parse/__init__.py@@ -1,43 +1,82 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0-"""Document parsers for various formats."""--from openviking.core.building_tree import BuildingTree-from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result-from openviking.parse.directory_scan import (- CLASS_PROCESSABLE,- CLASS_UNSUPPORTED,- ClassifiedFile,- DirectoryScanResult,- scan_directory,-)-from openviking.parse.mode import ParseMode, ParseModeInput-from openviking.parse.parsers.base_parser import BaseParser-from openviking.parse.parsers.html import HTMLParser-from openviking.parse.parsers.markdown import MarkdownParser-from openviking.parse.parsers.pdf import PDFParser-from openviking.parse.parsers.text import TextParser-from openviking.parse.tree_builder import TreeBuilder-from openviking.parse.vlm import VLMProcessor+"""Document parsers and their public lazy exports."""++from __future__ import annotations++from importlib import import_module+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from openviking.core.building_tree import BuildingTree+ from openviking.parse.base import NodeType, ParseResult, ResourceNode, create_parse_result+ from openviking.parse.directory_scan import (+ CLASS_PROCESSABLE,+ CLASS_UNSUPPORTED,+ ClassifiedFile,+ DirectoryScanResult,+ scan_directory,+ )+ from openviking.parse.mode import ParseMode, ParseModeInput+ from openviking.parse.parsers.base_parser import BaseParser+ from openviking.parse.parsers.html import HTMLParser+ from openviking.parse.parsers.markdown import MarkdownParser+ from openviking.parse.parsers.pdf import PDFParser+ from openviking.parse.parsers.text import TextParser+ from openviking.parse.tree_builder import TreeBuilder+ from openviking.parse.vlm import VLMProcessor++_EXPORTS = {+ "BaseParser": ("openviking.parse.parsers.base_parser", "BaseParser"),+ "ResourceNode": ("openviking.parse.base", "ResourceNode"),+ "NodeType": ("openviking.parse.base", "NodeType"),+ "ParseResult": ("openviking.parse.base", "ParseResult"),+ "create_parse_result": ("openviking.parse.base", "create_parse_result"),+ "TextParser": ("openviking.parse.parsers.text", "TextParser"),+ "MarkdownParser": ("openviking.parse.parsers.markdown", "MarkdownParser"),+ "PDFParser": ("openviking.parse.parsers.pdf", "PDFParser"),+ "HTMLParser": ("openviking.parse.parsers.html", "HTMLParser"),+ "TreeBuilder": ("openviking.parse.tree_builder", "TreeBuilder"),+ "BuildingTree": ("openviking.core.building_tree", "BuildingTree"),+ "VLMProcessor": ("openviking.parse.vlm", "VLMProcessor"),+ "CLASS_PROCESSABLE": ("openviking.parse.directory_scan", "CLASS_PROCESSABLE"),+ "CLASS_UNSUPPORTED": ("openviking.parse.directory_scan", "CLASS_UNSUPPORTED"),+ "ClassifiedFile": ("openviking.parse.directory_scan", "ClassifiedFile"),+ "DirectoryScanResult": ("openviking.parse.directory_scan", "DirectoryScanResult"),+ "scan_directory": ("openviking.parse.directory_scan", "scan_directory"),+ "ParseMode": ("openviking.parse.mode", "ParseMode"),+ "ParseModeInput": ("openviking.parse.mode", "ParseModeInput"),+}+++def __getattr__(name: str) -> Any:+ try:+ module_name, attr_name = _EXPORTS[name]+ except KeyError as exc:+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from exc++ value = getattr(import_module(module_name), attr_name)+ globals()[name] = value+ return value+++def __dir__() -> list[str]:+ return sorted(list(globals().keys()) + list(__all__))+__all__ = [- # Base classes and helpers"BaseParser","ResourceNode","NodeType","ParseResult","create_parse_result",- # Document parsers (core)"TextParser","MarkdownParser","PDFParser","HTMLParser",- # Tree builder"TreeBuilder","BuildingTree",- # VLM"VLMProcessor",- # Directory scan (phase-one validation)"CLASS_PROCESSABLE","CLASS_UNSUPPORTED","ClassifiedFile",diff --git a/openviking/parse/accessors/git_accessor.py b/openviking/parse/accessors/git_accessor.pyindex dad21ac81..cf46230ca 100644--- a/openviking/parse/accessors/git_accessor.py+++ b/openviking/parse/accessors/git_accessor.py@@ -19,6 +19,13 @@ from pathlib import Path, PurePosixPathfrom typing import Optional, Tuple, Unionfrom urllib.parse import quote, urlparse+from openviking.parse.network import (+ ParserNetworkContext,+ ParserNetworkOperation,+ open_parser_url,+ overlay_parser_subprocess_env,+)+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.utils import (is_github_url,is_gitlab_url,@@ -35,7 +42,6 @@ from openviking.utils.git_auth import (parse_git_http_auth_config,raise_git_auth_error,)-from openviking.parse.temp import get_parser_download_temp_dirfrom openviking_cli.utils.logger import get_loggerfrom .base import DataAccessor, LocalResource, SourceType@@ -390,6 +396,10 @@ class GitAccessor(DataAccessor):"""Clone a git repository into target_dir; return the repo name."""name = self._get_repo_name(url)logger.info(f"[GitAccessor] Cloning {url} to {target_dir}...")+ network_env = overlay_parser_subprocess_env(+ ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, url),+ env,+ )clone_args = ["git",@@ -401,19 +411,19 @@ class GitAccessor(DataAccessor):if branch and not commit:clone_args.extend(["--branch", branch])clone_args.extend([url, target_dir])- await self._run_git(clone_args, env=env)+ await self._run_git(clone_args, env=network_env)if commit:try:await self._run_git(["git", "-C", target_dir, "fetch", "origin", commit],- env=env,+ env=network_env,)except RuntimeError:try:await self._run_git(["git", "-C", target_dir, "fetch", "--all", "--tags", "--prune"],- env=env,+ env=network_env,)except RuntimeError:pass@@ -422,7 +432,7 @@ class GitAccessor(DataAccessor):try:await self._run_git(["git", "-C", target_dir, "fetch", "--unshallow", "origin"],- env=env,+ env=network_env,)except RuntimeError:pass@@ -437,7 +447,7 @@ class GitAccessor(DataAccessor):"origin","+refs/heads/*:refs/remotes/origin/*",],- env=env,+ env=network_env,)ok = await self._has_commit(target_dir, commit, env=env)if not ok:@@ -493,7 +503,8 @@ class GitAccessor(DataAccessor):headers["Authorization"] = f"token {github_token}"req = urllib.request.Request(zip_url, headers=headers)- with urllib.request.urlopen(req, timeout=1800) as resp, open(zip_path, "wb") as f:+ context = ParserNetworkContext(ParserNetworkOperation.GIT_ARCHIVE, zip_url)+ with open_parser_url(context, req, timeout=1800) as resp, open(zip_path, "wb") as f:shutil.copyfileobj(resp, f)try:@@ -586,7 +597,8 @@ class GitAccessor(DataAccessor):headers = {"User-Agent": "OpenViking"}req = urllib.request.Request(zip_url, headers=headers)- with urllib.request.urlopen(req, timeout=1800) as resp, open(zip_path, "wb") as f:+ context = ParserNetworkContext(ParserNetworkOperation.GIT_ARCHIVE, zip_url)+ with open_parser_url(context, req, timeout=1800) as resp, open(zip_path, "wb") as f:shutil.copyfileobj(resp, f)try:diff --git a/openviking/parse/accessors/http_accessor.py b/openviking/parse/accessors/http_accessor.pyindex d94ba2034..22b9fe2fa 100644--- a/openviking/parse/accessors/http_accessor.py+++ b/openviking/parse/accessors/http_accessor.py@@ -24,13 +24,18 @@ from typing import Any, Dict, Mapping, Optional, Tuple, Unionfrom urllib.parse import unquote, urlparsefrom openviking.parse.base import lazy_import-from openviking.parse.temp import get_parser_download_temp_dir+from openviking.parse.network import (+ ParserNetworkContext,+ ParserNetworkOperation,+ create_parser_httpx_client,+)from openviking.parse.parsers.constants import CODE_EXTENSIONSfrom openviking.parse.parsers.media.constants import (AUDIO_EXTENSIONS,IMAGE_EXTENSIONS,VIDEO_EXTENSIONS,)+from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.utils import is_code_hosting_blob_urlfrom openviking.utils.exceptions import error_code_from_http_statusfrom openviking.utils.network_guard import build_httpx_request_validation_hooks@@ -215,8 +220,13 @@ class URLTypeDetector:client_kwargs["event_hooks"] = event_hooksclient_kwargs["trust_env"] = False- async with httpx.AsyncClient(**client_kwargs) as client:- response = await client.head(url, headers=headers)+ client = create_parser_httpx_client(+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),+ client_cls=httpx.AsyncClient,+ **client_kwargs,+ )+ async with client as active_client:+ response = await active_client.head(url, headers=headers)meta["status_code"] = response.status_codeif not (200 <= response.status_code < 300):@@ -584,10 +594,15 @@ class HTTPAccessor(DataAccessor):client_kwargs["event_hooks"] = event_hooksclient_kwargs["trust_env"] = False- async with httpx.AsyncClient(**client_kwargs) as client:+ client = create_parser_httpx_client(+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),+ client_cls=httpx.AsyncClient,+ **client_kwargs,+ )+ async with client as active_client:headers = self._request_headers(tos_signature, tos_access)try:- response = await client.get(url, headers=headers)+ response = await active_client.get(url, headers=headers)response.raise_for_status()except httpx.ConnectError as e:cause: BaseException | None = ediff --git a/openviking/parse/accessors/web_crawler/middlewares.py b/openviking/parse/accessors/web_crawler/middlewares.pyindex b9ae22998..c01b092b7 100644--- a/openviking/parse/accessors/web_crawler/middlewares.py+++ b/openviking/parse/accessors/web_crawler/middlewares.py@@ -2,8 +2,15 @@# SPDX-License-Identifier: AGPL-3.0"""Scrapy downloader middlewares for the recursive web crawler."""+import ipaddress+from collections.abc import Mapping+from urllib.parse import urlparse+from urllib.request import proxy_bypass_environment+from scrapy.exceptions import IgnoreRequest+_PARSER_PROXY_MANAGED = "_openviking_parser_proxy_managed"+class RequestValidatorMiddleware:"""Pre-flight SSRF guard: validate every outbound URL, including redirects/retries."""@@ -17,3 +24,59 @@ class RequestValidatorMiddleware:except Exception as exc:raise IgnoreRequest(f"Blocked by request_validator: {exc}") from excreturn None+++def _proxy_value(environment: Mapping[str, str], scheme: str) -> str | None:+ return environment.get(f"{scheme}_proxy") or environment.get(f"{scheme.upper()}_PROXY")+++def _bypasses_proxy(environment: Mapping[str, str], host: str, port: int | None) -> bool:+ no_proxy = _proxy_value(environment, "no")+ if not no_proxy:+ return False+ host_with_port = f"{host}:{port}" if port is not None else host+ if proxy_bypass_environment(host_with_port, {"no": no_proxy}):+ return True+ try:+ address = ipaddress.ip_address(host)+ except ValueError:+ return False+ for value in no_proxy.split(","):+ try:+ if address in ipaddress.ip_network(value.strip(), strict=False):+ return True+ except ValueError:+ continue+ return False+++class ParserNetworkProxyMiddleware:+ """Apply the parser adapter's child-only proxy policy to each Scrapy request."""++ def process_request(self, request, spider):+ # Scrapy copies request.meta when following redirects. Keep a marker+ # for requests whose proxy was selected here so a redirect can choose+ # the proxy for its *new* scheme instead of reusing the old one. A+ # caller-supplied proxy remains authoritative and is never rewritten.+ managed = bool(request.meta.get(_PARSER_PROXY_MANAGED))+ if "proxy" in request.meta and not managed:+ return None+ environment = getattr(spider, "network_env", None) or {}+ parsed = urlparse(request.url)+ if parsed.scheme not in {"http", "https"} or not parsed.hostname:+ return None+ if not any(_proxy_value(environment, scheme) for scheme in ("http", "https", "all")):+ # No parser proxy was supplied. Leave Scrapy's normal+ # HttpProxyMiddleware behavior unchanged for deployments that do+ # not install the adapter.+ return None+ request.meta[_PARSER_PROXY_MANAGED] = True+ if _bypasses_proxy(environment, parsed.hostname, parsed.port):+ # ``HttpProxyMiddleware`` runs later and otherwise sees the child+ # proxy environment and installs a proxy again (notably for CIDR+ # NO_PROXY entries). A None proxy is its documented opt-out.+ request.meta["proxy"] = None+ return None+ proxy = _proxy_value(environment, parsed.scheme) or _proxy_value(environment, "all")+ request.meta["proxy"] = proxy+ return Nonediff --git a/openviking/parse/accessors/web_crawler/scrapy_spider.py b/openviking/parse/accessors/web_crawler/scrapy_spider.pyindex 9cf344242..39deccefd 100644--- a/openviking/parse/accessors/web_crawler/scrapy_spider.py+++ b/openviking/parse/accessors/web_crawler/scrapy_spider.py@@ -4,6 +4,7 @@import osfrom collections.abc import Iterator+from typing import Mappingfrom urllib.parse import urljoin, urlparseimport scrapy@@ -14,7 +15,6 @@ from openviking.parse.accessors.http_accessor import URLType, URLTypeDetectorfrom openviking.parse.accessors.web_crawler.config import CrawlConfigfrom openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage-_DOWNLOAD_URL_TYPES = frozenset({URLType.DOWNLOAD_PDF,@@ -64,6 +64,7 @@ class OpenVikingWebSpider(scrapy.Spider):config: CrawlConfig,collector: list[CrawledPage],download_collector: list[CrawledDownload],+ network_env: Mapping[str, str] | None = None,**kwargs,):super().__init__(**kwargs)@@ -71,6 +72,7 @@ class OpenVikingWebSpider(scrapy.Spider):self.config = configself.collector = collectorself.download_collector = download_collector+ self.network_env = dict(network_env or {})self.root_host = urlparse(root_url).netloc.lower()self._success_count = 0self._seen_download_urls: set[str] = set()diff --git a/openviking/parse/accessors/web_crawler/web_crawler.py b/openviking/parse/accessors/web_crawler/web_crawler.pyindex 05bb2e46e..0e449fa40 100644--- a/openviking/parse/accessors/web_crawler/web_crawler.py+++ b/openviking/parse/accessors/web_crawler/web_crawler.py@@ -4,7 +4,10 @@import asyncioimport multiprocessing+import osimport queue+from contextlib import contextmanager+from typing import Iterator, Mappingfrom openviking.parse.accessors.web_crawler.config import CrawlConfigfrom openviking.parse.accessors.web_crawler.models import (@@ -12,9 +15,45 @@ from openviking.parse.accessors.web_crawler.models import (CrawledPage,CrawlResult,)+from openviking.parse.network import (+ ParserNetworkContext,+ ParserNetworkOperation,+ overlay_parser_subprocess_env,+)+++@contextmanager+def _temporary_environment(overlay: Mapping[str, str]) -> Iterator[None]:+ """Install child-only network variables without leaking across worker reuse."""+ previous = {name: os.environ.get(name) for name in overlay}+ os.environ.update(overlay)+ try:+ yield+ finally:+ for name, value in previous.items():+ if value is None:+ os.environ.pop(name, None)+ else:+ os.environ[name] = value-def _run_crawl_worker(root_url: str, config: CrawlConfig, result_queue) -> None:++def _run_crawl_worker(+ root_url: str,+ config: CrawlConfig,+ result_queue,+ network_env: Mapping[str, str] | None = None,+) -> None:+ with _temporary_environment(network_env or {}):+ _run_crawl_worker_with_environment(root_url, config, result_queue, network_env or {})+++def _run_crawl_worker_with_environment(+ root_url: str,+ config: CrawlConfig,+ result_queue,+ network_env: Mapping[str, str],+) -> None:from scrapy.utils.reactor import install_reactorinstall_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")@@ -33,6 +72,7 @@ def _run_crawl_worker(root_url: str, config: CrawlConfig, result_queue) -> None:config=config,collector=pages,download_collector=downloads,+ network_env=network_env,)try:process.start()@@ -67,8 +107,9 @@ def _build_settings(config: CrawlConfig):"TELNETCONSOLE_ENABLED": False,"USER_AGENT": "OpenViking/0.4 (+recursive-web-crawler)","DOWNLOADER_MIDDLEWARES": {- "openviking.parse.accessors.web_crawler.middlewares."- "RequestValidatorMiddleware": 50,+ "openviking.parse.accessors.web_crawler.middlewares.RequestValidatorMiddleware": 50,+ "openviking.parse.accessors.web_crawler.middlewares.ParserNetworkProxyMiddleware": 100,+ "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,},})@@ -81,9 +122,13 @@ class ScrapyWebCrawler:async def crawl(self, root_url: str) -> CrawlResult:context = multiprocessing.get_context("spawn")result_queue = context.Queue()+ network_env = overlay_parser_subprocess_env(+ ParserNetworkContext(ParserNetworkOperation.WEB_CRAWL, root_url),+ {},+ )process = context.Process(target=_run_crawl_worker,- args=(root_url, self.config, result_queue),+ args=(root_url, self.config, result_queue, network_env),)process.start()try:diff --git a/openviking/parse/accessors/web_feed_accessor.py b/openviking/parse/accessors/web_feed_accessor.pyindex c9c9cc21c..d289e9265 100644--- a/openviking/parse/accessors/web_feed_accessor.py+++ b/openviking/parse/accessors/web_feed_accessor.py@@ -37,6 +37,11 @@ from urllib.parse import unquote, urljoin, urlparsefrom urllib.robotparser import RobotFileParserfrom openviking.parse.base import lazy_import+from openviking.parse.network import (+ ParserNetworkContext,+ ParserNetworkOperation,+ create_parser_httpx_client,+)from openviking.parse.temp import get_parser_download_temp_dirfrom openviking.utils.network_guard import build_httpx_request_validation_hooksfrom openviking_cli.utils.logger import get_logger@@ -297,19 +302,24 @@ class WebFeedAccessor(DataAccessor):try:client_kwargs = _build_httpx_client_kwargs(request_validator, cfg["request_timeout"])httpx = lazy_import("httpx")- async with httpx.AsyncClient(**client_kwargs) as client:+ client = create_parser_httpx_client(+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, source_url),+ client_cls=httpx.AsyncClient,+ **client_kwargs,+ )+ async with client as active_client:# The given URL may already be a sitemap/feed, or it may be a plain# page / bare domain (e.g. args={"site": True} on "https://t0saki.com").# In the latter case, auto-discover the site's sitemap/RSS.- feed_url, prefetched = await self._resolve_feed_source(source_url, client)+ feed_url, prefetched = await self._resolve_feed_source(source_url, active_client)entries, feed_kind = await self._collect(- feed_url, client, cfg, prefetched=prefetched+ feed_url, active_client, cfg, prefetched=prefetched)if not entries:raise RuntimeError(f"No pages found in sitemap/feed: {feed_url}")entries = self._filter_entries(entries, feed_url, cfg, include, exclude)- entries = await self._apply_robots(entries, feed_url, client, cfg)+ entries = await self._apply_robots(entries, feed_url, active_client, cfg)if len(entries) > cfg["max_pages"]:logger.warning(@@ -323,7 +333,7 @@ class WebFeedAccessor(DataAccessor):if not entries:raise RuntimeError(f"No ingestible pages remained after filtering: {feed_url}")- pages = await self._mirror(entries, client, temp_dir, cfg)+ pages = await self._mirror(entries, active_client, temp_dir, cfg)if not pages:raise RuntimeError(f"Failed to fetch any page from: {feed_url}")@@ -793,11 +803,16 @@ async def _discover_feed_hint(url: str, request_validator) -> Optional[str]:httpx = lazy_import("httpx")client_kwargs = _build_httpx_client_kwargs(request_validator, 5.0)- async with httpx.AsyncClient(**client_kwargs) as client:+ client = create_parser_httpx_client(+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, url),+ client_cls=httpx.AsyncClient,+ **client_kwargs,+ )+ async with client as active_client:# Fetch the page once for <link> autodiscovery (best-effort).page_content = b""try:- response = await client.get(url, headers={"User-Agent": _USER_AGENT})+ response = await active_client.get(url, headers={"User-Agent": _USER_AGENT})if (response.status_code < 400and "html" in response.headers.get("content-type", "").lower()@@ -806,9 +821,9 @@ async def _discover_feed_hint(url: str, request_validator) -> Optional[str]:except Exception:pass- for candidate in await _gather_feed_candidates(client, url, page_content):+ for candidate in await _gather_feed_candidates(active_client, url, page_content):try:- response = await client.get(candidate, headers={"User-Agent": _USER_AGENT})+ response = await active_client.get(candidate, headers={"User-Agent": _USER_AGENT})if response.status_code >= 400:continueinfo = _classify_and_count(response.content)diff --git a/openviking/parse/network.py b/openviking/parse/network.pynew file mode 100644index 000000000..cd31483b4--- /dev/null+++ b/openviking/parse/network.py@@ -0,0 +1,163 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0+"""Stable network extension point for parser resource acquisition."""++from __future__ import annotations++import os+import urllib.request+from dataclasses import dataclass+from enum import Enum+from threading import RLock+from typing import Any, Mapping, Protocol++import httpx+++class ParserNetworkOperation(str, Enum):+ """Network operations whose routing may be extended by a distribution."""++ HTTP_RESOURCE = "http_resource"+ GIT_ARCHIVE = "git_archive"+ GIT_COMMAND = "git_command"+ WEB_CRAWL = "web_crawl"+++@dataclass(frozen=True)+class ParserNetworkContext:+ """Strongly typed description of one parser network boundary."""++ operation: ParserNetworkOperation+ source_url: str++ def __post_init__(self) -> None:+ if not isinstance(self.operation, ParserNetworkOperation):+ raise TypeError("operation must be a ParserNetworkOperation")+ if not isinstance(self.source_url, str) or not self.source_url.strip():+ raise ValueError("source_url must be a non-empty string")+++class ParserNetworkAdapter(Protocol):+ """Distribution-owned routing hooks for parser network operations."""++ def supports(self, context: ParserNetworkContext) -> bool: ...++ def create_httpx_transport(+ self,+ context: ParserNetworkContext,+ ) -> httpx.AsyncBaseTransport | None: ...++ def create_urllib_opener(+ self,+ context: ParserNetworkContext,+ ) -> urllib.request.OpenerDirector | None: ...++ def subprocess_env_overlay(+ self,+ context: ParserNetworkContext,+ ) -> Mapping[str, str]: ...+++_adapter: ParserNetworkAdapter | None = None+_lock = RLock()+++def register_parser_network_adapter(adapter: ParserNetworkAdapter) -> None:+ """Register one process-wide parser network adapter.++ Re-registering the same object is idempotent. Replacing it is rejected so+ unrelated extensions cannot silently take ownership of parser egress.+ """++ global _adapter+ with _lock:+ if _adapter is adapter:+ return+ if _adapter is not None:+ raise RuntimeError("A parser network adapter is already registered")+ _adapter = adapter+++def reset_parser_network_adapter() -> None:+ """Reset the adapter. Intended for tests and runtime reinitialization."""++ global _adapter+ with _lock:+ _adapter = None+++def _adapter_for(context: ParserNetworkContext) -> ParserNetworkAdapter | None:+ with _lock:+ adapter = _adapter+ if adapter is not None and adapter.supports(context):+ return adapter+ return None+++def create_parser_httpx_client(+ context: ParserNetworkContext,+ *,+ client_cls: type[httpx.AsyncClient] = httpx.AsyncClient,+ **client_kwargs: Any,+) -> httpx.AsyncClient:+ """Create a parser HTTP client while preserving caller-owned safeguards."""++ adapter = _adapter_for(context)+ if adapter is not None:+ transport = adapter.create_httpx_transport(context)+ if transport is not None:+ if client_kwargs.get("transport") is not None:+ raise RuntimeError("Parser network adapter cannot replace an explicit transport")+ client_kwargs["transport"] = transport+ # The adapter transport owns proxy/direct routing. Environment+ # lookup here would reintroduce process-wide proxy side effects.+ client_kwargs["trust_env"] = False+ return client_cls(**client_kwargs)+++def open_parser_url(+ context: ParserNetworkContext,+ request: urllib.request.Request | str,+ **open_kwargs: Any,+):+ """Open one parser URL through an adapter opener when available."""++ adapter = _adapter_for(context)+ if adapter is not None:+ opener = adapter.create_urllib_opener(context)+ if opener is not None:+ return opener.open(request, **open_kwargs)+ return urllib.request.urlopen(request, **open_kwargs)+++def overlay_parser_subprocess_env(+ context: ParserNetworkContext,+ base_env: Mapping[str, str] | None,+) -> dict[str, str] | None:+ """Merge adapter variables into a child environment without mutating it."""++ adapter = _adapter_for(context)+ if adapter is None:+ return base_env if isinstance(base_env, dict) or base_env is None else dict(base_env)++ overlay = dict(adapter.subprocess_env_overlay(context))+ if not overlay:+ return base_env if isinstance(base_env, dict) or base_env is None else dict(base_env)+ if not all(isinstance(key, str) and isinstance(value, str) for key, value in overlay.items()):+ raise TypeError("Parser subprocess environment overlay must contain string pairs")++ merged = dict(os.environ if base_env is None else base_env)+ merged.update(overlay)+ return merged+++__all__ = [+ "ParserNetworkAdapter",+ "ParserNetworkContext",+ "ParserNetworkOperation",+ "create_parser_httpx_client",+ "open_parser_url",+ "overlay_parser_subprocess_env",+ "register_parser_network_adapter",+ "reset_parser_network_adapter",+]diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.pyindex df8a3c1c1..a607fe7fc 100644--- a/openviking/service/resource_service.py+++ b/openviking/service/resource_service.py@@ -30,6 +30,11 @@ from openviking.core.namespace import is_content_root_urifrom openviking.observability.http_error_context import sanitize_public_http_errorfrom openviking.parse.backend import ParserBackend, normalize_parser_backendfrom openviking.parse.mode import ParseMode, normalize_parse_mode+from openviking.parse.network import (+ ParserNetworkContext,+ ParserNetworkOperation,+ overlay_parser_subprocess_env,+)from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIASfrom openviking.resource.feishu_watch_auth import (FEISHU_ACCESS_TOKEN_ARG,@@ -1488,6 +1493,10 @@ class ResourceService:proc = Nonetry:env = build_git_http_auth_env(auth_config, source) if auth_config is not None else None+ env = overlay_parser_subprocess_env(+ ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, source),+ env,+ )proc = await asyncio.create_subprocess_exec("git","ls-remote",diff --git a/tests/misc/test_network_guard.py b/tests/misc/test_network_guard.pyindex ac7cd2be3..8be5a8bd6 100644--- a/tests/misc/test_network_guard.py+++ b/tests/misc/test_network_guard.py@@ -5,10 +5,18 @@from __future__ import annotationsfrom types import SimpleNamespace-from unittest.mock import AsyncMock, patch+from unittest.mock import patch+import httpximport pytest+from openviking.parse.network import (+ ParserNetworkContext,+ ParserNetworkOperation,+ create_parser_httpx_client,+ register_parser_network_adapter,+ reset_parser_network_adapter,+)from openviking.utils.network_guard import (_get_allowed_code_hosting_domains,_is_public_ip,@@ -287,48 +295,43 @@ class TestEnsurePublicRemoteTarget:class TestBuildHttpxRequestValidationHooks:- """Verify httpx hook construction."""+ """Verify optional validation and enforcement during parser downloads."""def test_returns_none_when_no_validator(self) -> None:assert build_httpx_request_validation_hooks(None) is None- def test_returns_request_hook_dict(self) -> None:- def dummy_validator(url: str) -> None:- pass-- hooks = build_httpx_request_validation_hooks(dummy_validator)- assert hooks is not None- assert "request" in hooks- assert len(hooks["request"]) == 1-@pytest.mark.asyncio- async def test_hook_calls_validator_with_url(self) -> None:- calls: list[str] = []-- def tracking_validator(url: str) -> None:- calls.append(url)-- hooks = build_httpx_request_validation_hooks(tracking_validator)- assert hooks is not None-- mock_request = AsyncMock()- mock_request.url = "http://example.com/test"-- hook_fn = hooks["request"][0]- await hook_fn(mock_request)-- assert calls == ["http://example.com/test"]-- @pytest.mark.asyncio- async def test_hook_propagates_validator_exception(self) -> None:- def failing_validator(url: str) -> None:- raise PermissionDeniedError("blocked")-- hooks = build_httpx_request_validation_hooks(failing_validator)- assert hooks is not None-- mock_request = AsyncMock()- mock_request.url = "http://evil.com"-- with pytest.raises(PermissionDeniedError, match="blocked"):- await hooks["request"][0](mock_request)+ async def test_parser_adapter_preserves_redirect_validation(self) -> None:+ source = "https://example.com/document.txt"+ private_target = "http://127.0.0.1/admin"+ validated = []+ downloaded = []++ def validate(url):+ validated.append(url)+ if url == private_target:+ raise PermissionDeniedError("blocked")++ async def download(request):+ downloaded.append(str(request.url))+ return httpx.Response(302, headers={"Location": private_target})++ register_parser_network_adapter(+ SimpleNamespace(+ supports=lambda context: context.operation is ParserNetworkOperation.HTTP_RESOURCE,+ create_httpx_transport=lambda context: httpx.MockTransport(download),+ )+ )+ try:+ async with create_parser_httpx_client(+ ParserNetworkContext(ParserNetworkOperation.HTTP_RESOURCE, source),+ follow_redirects=True,+ event_hooks=build_httpx_request_validation_hooks(validate),+ ) as client:+ with pytest.raises(PermissionDeniedError, match="blocked"):+ await client.get(source)+ finally:+ reset_parser_network_adapter()++ assert validated == [source, private_target]+ assert downloaded == [source]diff --git a/tests/parse/accessors/web_crawler/test_middlewares.py b/tests/parse/accessors/web_crawler/test_middlewares.pyindex efd7d0265..6b6010240 100644--- a/tests/parse/accessors/web_crawler/test_middlewares.py+++ b/tests/parse/accessors/web_crawler/test_middlewares.py@@ -1,53 +1,75 @@-from unittest.mock import MagicMock+from types import SimpleNamespace+from unittest.mock import patchimport pytest+from scrapy import Request, Spider+from scrapy.crawler import Crawler+from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware+from scrapy.downloadermiddlewares.redirect import RedirectMiddlewarefrom scrapy.exceptions import IgnoreRequest+from scrapy.http import Response+from scrapy.settings import Settingsfrom openviking.parse.accessors.web_crawler.config import CrawlConfig-from openviking.parse.accessors.web_crawler.middlewares import RequestValidatorMiddleware+from openviking.parse.accessors.web_crawler.middlewares import (+ ParserNetworkProxyMiddleware,+ RequestValidatorMiddleware,+)-def _make_spider(validator):- spider = MagicMock()- spider.config = CrawlConfig(request_validator=validator)- return spider+class TestParserRequestRouting:+ def test_raises_ignore_when_validator_rejects(self):+ def reject(url):+ raise ValueError(f"private address: {url}")+ spider = SimpleNamespace(config=CrawlConfig(request_validator=reject))+ request = Request("http://169.254.169.254/latest/meta-data/")+ with pytest.raises(IgnoreRequest, match="private address"):+ RequestValidatorMiddleware().process_request(request, spider)-def _make_request(url):- request = MagicMock()- request.url = url- return request+ def test_redirects_revalidate_and_select_proxy_or_direct_route(self):+ environment = {+ "HTTP_PROXY": "http://http-proxy.example:8118",+ "HTTPS_PROXY": "http://https-proxy.example:8118",+ "NO_PROXY": "93.184.0.0/16",+ }+ seen = []+ spider = SimpleNamespace(+ config=CrawlConfig(request_validator=seen.append),+ network_env=environment,+ )+ validator = RequestValidatorMiddleware()+ routing = ParserNetworkProxyMiddleware()+ redirects = RedirectMiddleware.from_crawler(Crawler(Spider, Settings()))+ with patch.dict("os.environ", environment, clear=True):+ native_proxy = HttpProxyMiddleware()+ request = Request("http://example.com/start")+ validator.process_request(request, spider)+ routing.process_request(request, spider)+ native_proxy.process_request(request, spider)+ assert request.meta["proxy"] == environment["HTTP_PROXY"]-class TestRequestValidatorMiddleware:- def test_passes_through_when_validator_accepts(self):- mw = RequestValidatorMiddleware()- spider = _make_spider(validator=lambda _url: None)- request = _make_request("https://example.com/")- assert mw.process_request(request, spider) is None+ response = Response(+ request.url, status=301, headers={"Location": "https://example.com/final"}+ )+ request = redirects.process_response(request, response, spider)+ validator.process_request(request, spider)+ routing.process_request(request, spider)+ native_proxy.process_request(request, spider)+ assert request.meta["proxy"] == environment["HTTPS_PROXY"]- def test_raises_ignore_when_validator_rejects(self):- mw = RequestValidatorMiddleware()-- def reject(url):- raise ValueError(f"private address: {url}")+ response = Response(+ request.url, status=302, headers={"Location": "http://93.184.216.34/article"}+ )+ request = redirects.process_response(request, response, spider)+ validator.process_request(request, spider)+ routing.process_request(request, spider)+ native_proxy.process_request(request, spider)+ assert request.meta["proxy"] is None- spider = _make_spider(validator=reject)- request = _make_request("http://169.254.169.254/latest/meta-data/")- with pytest.raises(IgnoreRequest) as exc_info:- mw.process_request(request, spider)- assert "private address" in str(exc_info.value)-- def test_no_op_when_validator_unset(self):- mw = RequestValidatorMiddleware()- spider = _make_spider(validator=None)- request = _make_request("https://example.com/")- assert mw.process_request(request, spider) is None-- def test_validator_receives_request_url(self):- mw = RequestValidatorMiddleware()- seen: list[str] = []- spider = _make_spider(validator=lambda url: seen.append(url))- request = _make_request("https://example.com/redirected-target")- mw.process_request(request, spider)- assert seen == ["https://example.com/redirected-target"]+ assert seen == [+ "http://example.com/start",+ "https://example.com/final",+ "http://93.184.216.34/article",+ ]diff --git a/tests/service/test_resource_service_git_preflight.py b/tests/service/test_resource_service_git_preflight.pyindex eb97ded11..da91054e1 100644--- a/tests/service/test_resource_service_git_preflight.py+++ b/tests/service/test_resource_service_git_preflight.py@@ -3,6 +3,7 @@"""Regression tests for Git preflight process cleanup."""import asyncio+import base64import osimport signalfrom types import SimpleNamespace@@ -11,6 +12,11 @@ from unittest.mock import AsyncMock, Mock, callimport pytestfrom openviking.parse.mode import ParseMode+from openviking.parse.network import (+ ParserNetworkOperation,+ register_parser_network_adapter,+ reset_parser_network_adapter,+)from openviking.server.identity import RequestContext, Rolefrom openviking.service import resource_service as resource_service_modulefrom openviking.service.resource_service import ResourceService@@ -20,16 +26,16 @@ from openviking_cli.session.user_id import UserIdentifierpytestmark = pytest.mark.skipif(os.name != "posix", reason="process groups are POSIX-only")-async def test_account_github_token_is_used_for_repository_preflight(monkeypatch):+async def test_account_token_and_parser_proxy_reach_repository_preflight(monkeypatch):token_resolver = AsyncMock(return_value="account-token")processor = SimpleNamespace(github_token_for=token_resolver)service = ResourceService(resource_processor=processor)- service._preflight_git_source = AsyncMock(- return_value=SimpleNamespace(- source_name="private",- source_path="https://github.com/org/private",- source_format="repository",- )+ process = Mock(returncode=0)+ process.communicate = AsyncMock(return_value=(b"refs/heads/main\n", b""))+ execute = AsyncMock(return_value=process)+ monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)+ monkeypatch.setattr(+ resource_service_module, "parse_code_hosting_url", lambda _source: "org/private")monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)monkeypatch.setattr(resource_service_module, "is_github_url", lambda _path: True)@@ -38,19 +44,41 @@ async def test_account_github_token_is_used_for_repository_preflight(monkeypatchrole=Role.USER,)- plan = await service._prepare_standard_source_plan(- path="https://github.com/org/private",- ctx=ctx,- mode=ParseMode.DEFAULT,- allow_local_path_resolution=False,- processor_kwargs={},- watch_auth_state=None,+ register_parser_network_adapter(+ SimpleNamespace(+ supports=lambda context: context.operation is ParserNetworkOperation.GIT_COMMAND,+ subprocess_env_overlay=lambda context: {"HTTPS_PROXY": "http://proxy.example:8118"},+ ))+ try:+ plan = await service._prepare_standard_source_plan(+ path="https://github.com/org/private",+ ctx=ctx,+ mode=ParseMode.DEFAULT,+ allow_local_path_resolution=False,+ processor_kwargs={},+ watch_auth_state=None,+ )+ finally:+ reset_parser_network_adapter()token_resolver.assert_awaited_once_with("https://github.com/org/private", ctx)- preflight_auth = service._preflight_git_source.await_args.kwargs["auth_config"]- assert preflight_auth.username == "oauth2"- assert preflight_auth.token == "account-token"+ assert execute.await_args.args == (+ "git",+ "ls-remote",+ "--heads",+ "https://github.com/org/private",+ )+ env = execute.await_args.kwargs["env"]+ assert env["HTTPS_PROXY"] == "http://proxy.example:8118"+ entries = {+ env[f"GIT_CONFIG_KEY_{index}"]: env[f"GIT_CONFIG_VALUE_{index}"]+ for index in range(int(env["GIT_CONFIG_COUNT"]))+ }+ encoded = base64.b64encode(b"oauth2:account-token").decode("ascii")+ assert entries["http.https://github.com/org/private.extraHeader"] == (+ f"Authorization: Basic {encoded}"+ )assert plan is not Noneassert plan.task_auth == {}assert "account-token" not in str(plan.processor_args)