Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix(ingest): SSRF protection for Web and API ingestors (#867) (#906)
pravit-ampab5c12f9afc6
diff --git a/semantica/ingest/api_ingestor.py b/semantica/ingest/api_ingestor.pyindex 30677681..d1f7a860 100644--- a/semantica/ingest/api_ingestor.py+++ b/semantica/ingest/api_ingestor.py@@ -38,6 +38,7 @@ except (ImportError, OSError):from ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+from .ssrf import parse_bool, request_with_ssrf_guard@dataclass@@ -78,11 +79,16 @@ class RESTIngestor:Args:config: Optional REST API ingestion configuration dictionary- **kwargs: Additional configuration parameters (merged into config)+ **kwargs: Additional configuration parameters (merged into config).+ Recognized keys include ``allow_private_ips`` (default False) to+ opt into fetching private/loopback/link-local endpoints."""self.logger = get_logger("api_ingestor")self.config = config or {}self.config.update(kwargs)+ self.allow_private_ips = parse_bool(+ self.config.get("allow_private_ips"), default=False+ )# Initialize session with retry strategyself.session = requests.Session()@@ -103,7 +109,10 @@ class RESTIngestor:# Initialize progress trackerself.progress_tracker = get_progress_tracker()- self.logger.debug("REST API ingestor initialized")+ self.logger.debug(+ "REST API ingestor initialized (allow_private_ips=%s)",+ self.allow_private_ips,+ )def ingest_endpoint(self,@@ -137,7 +146,7 @@ class RESTIngestor:- metadata: Additional metadataRaises:- ValidationError: If endpoint is invalid+ ValidationError: If endpoint is invalid or fails SSRF checksProcessingError: If request fails"""tracking_id = self.progress_tracker.start_tracking(@@ -153,10 +162,12 @@ class RESTIngestor:if headers:request_headers.update(headers)- # Make request- response = self.session.request(- method=method,- url=endpoint,+ # Make request (SSRF-safe; validates URL and each redirect hop)+ response = request_with_ssrf_guard(+ method,+ endpoint,+ session=self.session,+ allow_private_ips=self.allow_private_ips,headers=request_headers,params=params,data=data,@@ -196,6 +207,11 @@ class RESTIngestor:},)+ except ValidationError:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message=f"Invalid endpoint URL: {endpoint}"+ )+ raiseexcept requests.exceptions.RequestException as e:self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.pyindex a3e3ba76..ac9775a2 100644--- a/semantica/ingest/methods.py+++ b/semantica/ingest/methods.py@@ -553,6 +553,12 @@ def ingest_web(# Get configconfig = ingest_config.get_method_config("web")config.update(kwargs)+ if "allow_private_ips" in config:+ from .ssrf import parse_bool++ config["allow_private_ips"] = parse_bool(+ config["allow_private_ips"], default=False+ )ingestor = WebIngestor(**config)diff --git a/semantica/ingest/ssrf.py b/semantica/ingest/ssrf.pynew file mode 100644index 00000000..ade93a46--- /dev/null+++ b/semantica/ingest/ssrf.py@@ -0,0 +1,293 @@+"""SSRF safeguards for ingest HTTP clients.++Validates outbound request URLs before they reach ``requests`` / urllib3 so+user-supplied targets cannot reach private, loopback, or link-local+addresses (including cloud metadata endpoints).+"""++from __future__ import annotations++import concurrent.futures+import ipaddress+import socket+import threading+from typing import Any, Iterable, Optional+from urllib.parse import urljoin, urlparse++import requests++from ..utils.exceptions import ValidationError++ALLOWED_URL_SCHEMES = frozenset({"http", "https"})++_TRUE_STRINGS = frozenset({"1", "true", "yes", "on"})+_FALSE_STRINGS = frozenset({"0", "false", "no", "off"})++# Keep DNS resolution bounded so fake/unreachable hosts in tests and offline+# environments cannot hang request validation indefinitely.+_DNS_RESOLVE_TIMEOUT_SECONDS = 2.0+_DNS_EXECUTOR_WORKERS = 4++# Bound manual redirect following so open redirect chains cannot hang fetches.+_DEFAULT_MAX_REDIRECTS = 10+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})+_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303})++_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None+_dns_executor_lock = threading.Lock()+++def parse_bool(value: Any, default: bool = False) -> bool:+ """Parse a config value into a bool without truthy-string pitfalls.++ ``bool('false')`` is ``True`` in Python, which would silently disable SSRF+ protections when string-typed config reaches ingestors. This helper accepts+ only explicit bools and a small allowlist of string/int forms.++ Args:+ value: Config value to interpret. ``None`` yields *default*.+ default: Value returned when *value* is ``None``.++ Raises:+ ValidationError: If *value* is not a recognized boolean form.+ """+ if value is None:+ return default+ if isinstance(value, bool):+ return value+ if isinstance(value, (int, float)):+ if value == 1:+ return True+ if value == 0:+ return False+ raise ValidationError(f"Invalid boolean value: {value!r}")+ if isinstance(value, str):+ normalized = value.strip().lower()+ if normalized in _TRUE_STRINGS:+ return True+ if normalized in _FALSE_STRINGS:+ return False+ raise ValidationError(f"Invalid boolean value: {value!r}")+ raise ValidationError(+ f"Invalid boolean type: {type(value).__name__} ({value!r})"+ )+++def _shutdown_executor(executor: concurrent.futures.ThreadPoolExecutor) -> None:+ """Shut down *executor* without waiting for in-flight DNS lookups."""+ try:+ executor.shutdown(wait=False, cancel_futures=True)+ except TypeError:+ # cancel_futures was added in Python 3.9.+ executor.shutdown(wait=False)+++def _get_dns_executor() -> concurrent.futures.ThreadPoolExecutor:+ """Return a process-wide executor for bounded DNS lookups."""+ global _dns_executor+ with _dns_executor_lock:+ if _dns_executor is None:+ _dns_executor = concurrent.futures.ThreadPoolExecutor(+ max_workers=_DNS_EXECUTOR_WORKERS,+ thread_name_prefix="semantica-ssrf-dns",+ )+ return _dns_executor+++# Explicit blocked networks from issue #867, plus common non-routable ranges.+BLOCKED_NETWORKS = (+ ipaddress.ip_network("0.0.0.0/8"),+ ipaddress.ip_network("10.0.0.0/8"),+ ipaddress.ip_network("127.0.0.0/8"),+ ipaddress.ip_network("169.254.0.0/16"), # link-local / cloud metadata+ ipaddress.ip_network("172.16.0.0/12"),+ ipaddress.ip_network("192.168.0.0/16"),+ ipaddress.ip_network("::1/128"),+ ipaddress.ip_network("fc00::/7"),+ ipaddress.ip_network("fe80::/10"),+)+++def _ip_is_blocked(addr: ipaddress._BaseAddress) -> bool:+ if (+ addr.is_private+ or addr.is_loopback+ or addr.is_link_local+ or addr.is_reserved+ or addr.is_multicast+ or addr.is_unspecified+ ):+ return True+ return any(addr in network for network in BLOCKED_NETWORKS)+++def _hostname_resolves_to_blocked(hostname: str) -> bool:+ """Return True if any resolved address for *hostname* is blocked.++ DNS lookups run on a shared thread pool with ``Future.result(timeout=...)``.+ Do not use ``with ThreadPoolExecutor(...)`` here: on timeout, leaving the+ context waits for the hung ``getaddrinfo`` worker and defeats the bound.++ Raises:+ ValidationError: If DNS resolution fails or times out. Fail closed so+ outbound requests never proceed without confirmed safe IPs.+ """+ executor = _get_dns_executor()+ owned_executor = False+ try:+ try:+ future = executor.submit(socket.getaddrinfo, hostname, None)+ except RuntimeError:+ # Shared executor was shut down; use a throwaway pool that never+ # blocks the caller on shutdown.+ executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)+ owned_executor = True+ future = executor.submit(socket.getaddrinfo, hostname, None)+ resolved: Iterable = future.result(timeout=_DNS_RESOLVE_TIMEOUT_SECONDS)+ except (socket.gaierror, concurrent.futures.TimeoutError, OSError) as exc:+ raise ValidationError(+ f"URL host '{hostname}' could not be resolved safely "+ "(DNS error or timeout); request blocked"+ ) from exc+ finally:+ if owned_executor:+ _shutdown_executor(executor)++ for info in resolved:+ sockaddr = info[4]+ addr = ipaddress.ip_address(sockaddr[0])+ if _ip_is_blocked(addr):+ return True+ return False+++def validate_url_for_request(+ url: str, *, allow_private_ips: bool = False+) -> None:+ """Validate that *url* is safe to fetch over HTTP(S).++ Args:+ url: Absolute URL to validate.+ allow_private_ips: When True, skip private/loopback/link-local checks+ (for trusted internal deployments).++ Raises:+ ValidationError: If the scheme is not http/https, the URL is malformed,+ or the host targets a blocked address space.+ """+ if not isinstance(url, str) or not url.strip():+ raise ValidationError("URL must be a non-empty string")++ parsed = urlparse(url.strip())+ scheme = (parsed.scheme or "").lower()+ if scheme not in ALLOWED_URL_SCHEMES:+ raise ValidationError(+ f"URL scheme '{parsed.scheme}' is not permitted. "+ "Only http and https are allowed."+ )+ if not parsed.netloc:+ raise ValidationError(+ f"Invalid URL format: {url}. "+ "URL must include scheme (http/https) and netloc (domain)."+ )++ host = parsed.hostname+ if not host:+ raise ValidationError(+ f"Invalid URL format: {url}. "+ "URL must include a hostname."+ )++ if allow_private_ips:+ return++ lowered = host.lower().rstrip(".")+ if lowered == "localhost" or lowered.endswith(".localhost"):+ raise ValidationError(f"URL host is not allowed: {host}")++ try:+ literal_ip = ipaddress.ip_address(host)+ except ValueError:+ literal_ip = None++ if literal_ip is not None:+ if _ip_is_blocked(literal_ip):+ raise ValidationError(f"URL points to a blocked address: {host}")+ return++ if _hostname_resolves_to_blocked(host):+ raise ValidationError(+ f"URL host '{host}' resolves to a blocked (private/loopback/"+ "link-local) address"+ )+++def request_with_ssrf_guard(+ method: str,+ url: str,+ *,+ session: Optional[requests.Session] = None,+ allow_private_ips: bool = False,+ max_redirects: int = _DEFAULT_MAX_REDIRECTS,+ **kwargs: Any,+) -> requests.Response:+ """Perform an HTTP request with SSRF checks on *url* and every redirect.++ ``requests`` follows redirects by default, which would allow a validated+ public URL to bounce into private/loopback/link-local space. This helper+ disables automatic redirects and re-validates each ``Location`` target+ before issuing the next hop.+ """+ kwargs = dict(kwargs)+ kwargs.pop("allow_redirects", None)++ validate_url_for_request(url, allow_private_ips=allow_private_ips)++ requester = session.request if session is not None else requests.request+ current_url = url+ current_method = method.upper()+ redirects_followed = 0++ while True:+ response = requester(+ current_method,+ current_url,+ allow_redirects=False,+ **kwargs,+ )++ if response.status_code not in _REDIRECT_STATUS_CODES:+ return response++ if redirects_followed >= max_redirects:+ response.close()+ raise ValidationError(+ f"Exceeded maximum redirects ({max_redirects}) while "+ f"fetching '{url}'"+ )++ location = response.headers.get("Location")+ if not location or not str(location).strip():+ response.close()+ raise ValidationError(+ f"Redirect from '{current_url}' is missing a Location header"+ )++ next_url = urljoin(current_url, str(location).strip())+ validate_url_for_request(next_url, allow_private_ips=allow_private_ips)++ # Match requests' historical method rewriting for 301/302/303.+ if (+ response.status_code in _STRIP_BODY_ON_REDIRECT+ and current_method not in {"GET", "HEAD"}+ ):+ current_method = "GET"+ for key in ("data", "json", "files"):+ kwargs.pop(key, None)++ # Params apply to the original request URL only; Location is authoritative.+ kwargs.pop("params", None)++ response.close()+ current_url = next_url+ redirects_followed += 1diff --git a/semantica/ingest/web_ingestor.py b/semantica/ingest/web_ingestor.pyindex e542ee39..5dd77f31 100644--- a/semantica/ingest/web_ingestor.py+++ b/semantica/ingest/web_ingestor.py@@ -48,6 +48,7 @@ from urllib3.util.retry import Retryfrom ..utils.exceptions import ProcessingError, ValidationErrorfrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_tracker+from .ssrf import parse_bool, request_with_ssrf_guard, validate_url_for_request@dataclass@@ -338,10 +339,14 @@ class SitemapCrawler:Sets up the crawler with configuration options.Args:- **config: Crawler configuration options (currently unused)+ **config: Crawler configuration options. Recognized keys:+ - allow_private_ips: Allow private/loopback sitemap hosts"""self.logger = get_logger("sitemap_crawler")self.config = config+ self.allow_private_ips = parse_bool(+ config.get("allow_private_ips"), default=False+ )def parse_sitemap(self, sitemap_url: str) -> List[str]:"""@@ -359,10 +364,16 @@ class SitemapCrawler:Raises:ProcessingError: If sitemap cannot be fetched or parsed+ ValidationError: If sitemap_url fails SSRF checks"""try:- # Fetch sitemap- response = requests.get(sitemap_url, timeout=30)+ # Fetch sitemap (SSRF-safe; validates URL and each redirect hop)+ response = request_with_ssrf_guard(+ "GET",+ sitemap_url,+ allow_private_ips=self.allow_private_ips,+ timeout=30,+ )response.raise_for_status()# Parse XML@@ -391,6 +402,8 @@ class SitemapCrawler:)return urls+ except ValidationError:+ raiseexcept Exception as e:self.logger.error(f"Failed to parse sitemap {sitemap_url}: {e}")raise ProcessingError(f"Failed to parse sitemap: {e}") from e@@ -411,10 +424,16 @@ class SitemapCrawler:Raises:ProcessingError: If sitemap index cannot be fetched or parsed+ ValidationError: If index_url fails SSRF checks"""try:- # Fetch sitemap index- response = requests.get(index_url, timeout=30)+ # Fetch sitemap index (SSRF-safe; validates URL and each redirect hop)+ response = request_with_ssrf_guard(+ "GET",+ index_url,+ allow_private_ips=self.allow_private_ips,+ timeout=30,+ )response.raise_for_status()# Parse XML@@ -448,6 +467,8 @@ class SitemapCrawler:)return all_urls+ except ValidationError:+ raiseexcept Exception as e:self.logger.error(f"Failed to crawl sitemap index {index_url}: {e}")raise ProcessingError(f"Failed to crawl sitemap index: {e}") from e@@ -487,6 +508,7 @@ class WebIngestor:max_retries: int = 3,backoff_factor: float = 1.0,timeout: int = 30,+ allow_private_ips: bool = False,config: Optional[Dict[str, Any]] = None,**kwargs,):@@ -503,12 +525,19 @@ class WebIngestor:max_retries: Maximum number of retry attempts (default: 3)backoff_factor: Backoff factor for retries (default: 1.0)timeout: Request timeout in seconds (default: 30)+ allow_private_ips: Allow fetching private/loopback/link-local hosts+ (default: False). Opt in only for trusted internal deployments.config: Optional configuration dictionary (merged with kwargs)**kwargs: Additional configuration parameters"""self.logger = get_logger("web_ingestor")self.config = config or {}self.config.update(kwargs)+ self.allow_private_ips = parse_bool(+ self.config.get("allow_private_ips", allow_private_ips),+ default=False,+ )+ self.config["allow_private_ips"] = self.allow_private_ips# Initialize HTTP session with retry strategyself.session = requests.Session()@@ -544,7 +573,8 @@ class WebIngestor:self.logger.debug(f"Web ingestor initialized: user_agent={user_agent}, "- f"delay={delay}, respect_robots={respect_robots}"+ f"delay={delay}, respect_robots={respect_robots}, "+ f"allow_private_ips={self.allow_private_ips}")def ingest_url(@@ -574,16 +604,9 @@ class WebIngestor:)try:- # Validate URL format- try:- parsed = urlparse(url)- if not parsed.scheme or not parsed.netloc:- raise ValidationError(- f"Invalid URL format: {url}. "- "URL must include scheme (http/https) and netloc (domain)."- )- except Exception as e:- raise ValidationError(f"Invalid URL: {url}") from e+ # Validate before robots check: RobotsChecker.read() makes an unguarded+ # HTTP request to <host>/robots.txt and must not reach blocked addresses.+ validate_url_for_request(url, allow_private_ips=self.allow_private_ips)# Check robots.txt complianceif self.robots_checker and not self.robots_checker.can_fetch(url):@@ -593,11 +616,19 @@ class WebIngestor:# Apply rate limiting (wait if necessary)self.rate_limiter.wait_if_needed()- # Fetch content with retry logic+ # Fetch content with retry logic (SSRF-safe redirects)try:request_timeout = timeout or self.config.get("timeout", 30)- response = self.session.get(url, timeout=request_timeout)+ response = request_with_ssrf_guard(+ "GET",+ url,+ session=self.session,+ allow_private_ips=self.allow_private_ips,+ timeout=request_timeout,+ )response.raise_for_status()+ except ValidationError:+ raiseexcept requests.RequestException as e:self.progress_tracker.stop_tracking(tracking_id, status="failed", message=str(e)diff --git a/tests/ingest/test_ssrf_protection.py b/tests/ingest/test_ssrf_protection.pynew file mode 100644index 00000000..2da84530--- /dev/null+++ b/tests/ingest/test_ssrf_protection.py@@ -0,0 +1,407 @@+"""SSRF protection tests for web and API ingestors (issue #867)."""++from unittest.mock import MagicMock, patch++import pytest+import urllib3.connectionpool as pool++from semantica.ingest.api_ingestor import RESTIngestor+from semantica.ingest.ssrf import (+ parse_bool,+ request_with_ssrf_guard,+ validate_url_for_request,+)+from semantica.ingest.web_ingestor import SitemapCrawler, WebIngestor+from semantica.utils.exceptions import ValidationError+++class TestParseBool:+ def test_bool_passthrough(self):+ assert parse_bool(True) is True+ assert parse_bool(False) is False++ def test_none_uses_default(self):+ assert parse_bool(None) is False+ assert parse_bool(None, default=True) is True++ def test_string_true_values(self):+ for value in ("true", "TRUE", "1", "yes", "on", " Yes "):+ assert parse_bool(value) is True++ def test_string_false_values(self):+ for value in ("false", "FALSE", "0", "no", "off", " No "):+ assert parse_bool(value) is False++ def test_int_zero_one(self):+ assert parse_bool(0) is False+ assert parse_bool(1) is True++ def test_rejects_unknown_string(self):+ with pytest.raises(ValidationError, match="Invalid boolean"):+ parse_bool("maybe")+++class TestValidateUrlForRequest:+ def test_accepts_https(self):+ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ validate_url_for_request("https://example.com/path")++ def test_rejects_file_scheme(self):+ with pytest.raises(ValidationError, match="not permitted"):+ validate_url_for_request("file://localhost/etc/passwd")++ def test_rejects_gopher_scheme(self):+ with pytest.raises(ValidationError, match="not permitted"):+ validate_url_for_request("gopher://example.com/1")++ def test_rejects_literal_private_ips(self):+ for url in (+ "http://10.0.0.1/",+ "http://192.168.1.1/",+ "http://172.16.5.5/internal",+ "http://127.0.0.1:9999/internal",+ "http://169.254.169.254/latest/meta-data/",+ ):+ with pytest.raises(ValidationError, match="blocked"):+ validate_url_for_request(url)++ def test_rejects_localhost_hostname(self):+ with pytest.raises(ValidationError, match="not allowed"):+ validate_url_for_request("http://localhost/admin")++ def test_rejects_hostname_resolving_to_private_ip(self):+ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("10.0.0.5", 0))],+ ):+ with pytest.raises(ValidationError, match="blocked"):+ validate_url_for_request("http://internal.corp/secret")++ def test_allow_private_ips_opt_in(self):+ validate_url_for_request(+ "http://127.0.0.1:8080/health", allow_private_ips=True+ )+ # Scheme allowlist still applies when private IPs are permitted+ with pytest.raises(ValidationError, match="not permitted"):+ validate_url_for_request(+ "file://localhost/x", allow_private_ips=True+ )++ def test_dns_failure_raises(self):+ import socket++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ side_effect=socket.gaierror("name or service not known"),+ ):+ with pytest.raises(ValidationError, match="could not be resolved safely"):+ validate_url_for_request("http://does-not-resolve.invalid/path")++ def test_dns_timeout_raises(self):+ import concurrent.futures++ with patch(+ "semantica.ingest.ssrf.concurrent.futures.Future.result",+ side_effect=concurrent.futures.TimeoutError(),+ ):+ with pytest.raises(ValidationError, match="could not be resolved safely"):+ validate_url_for_request("http://slow-dns.example/path")++ def test_hung_getaddrinfo_returns_within_timeout_bound(self):+ """Timeout must not wait on executor shutdown for a blocking resolver."""+ import time++ import semantica.ingest.ssrf as ssrf++ hang_seconds = 1.5+ bound = 0.1++ def hanging_getaddrinfo(*_args, **_kwargs):+ time.sleep(hang_seconds)+ return [(None, None, None, None, ("93.184.216.34", 0))]++ with patch.object(ssrf, "_DNS_RESOLVE_TIMEOUT_SECONDS", bound), patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ side_effect=hanging_getaddrinfo,+ ):+ start = time.monotonic()+ with pytest.raises(ValidationError, match="could not be resolved safely"):+ ssrf._hostname_resolves_to_blocked("hanging.example")+ elapsed = time.monotonic() - start++ # Must fail closed near the configured bound, not after hang_seconds+ # (which is what ``with ThreadPoolExecutor`` shutdown waiting causes).+ assert elapsed < hang_seconds / 2+ assert elapsed < bound + 0.75+++class TestRequestWithSsrfGuardRedirects:+ def test_blocks_redirect_to_loopback(self):+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "http://127.0.0.1/secret"}+ redirect.close = MagicMock()++ session = MagicMock()+ session.request.return_value = redirect++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ with pytest.raises(ValidationError, match="blocked"):+ request_with_ssrf_guard(+ "GET",+ "https://example.com/start",+ session=session,+ )++ session.request.assert_called_once()+ assert session.request.call_args.kwargs.get("allow_redirects") is False++ def test_blocks_redirect_to_metadata_ip(self):+ redirect = MagicMock()+ redirect.status_code = 301+ redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"}+ redirect.close = MagicMock()++ session = MagicMock()+ session.request.return_value = redirect++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ with pytest.raises(ValidationError, match="blocked"):+ request_with_ssrf_guard(+ "GET",+ "https://example.com/start",+ session=session,+ )++ def test_follows_safe_redirect(self):+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "https://example.com/final"}+ redirect.close = MagicMock()++ final = MagicMock()+ final.status_code = 200+ final.headers = {}++ session = MagicMock()+ session.request.side_effect = [redirect, final]++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ response = request_with_ssrf_guard(+ "GET",+ "https://example.com/start",+ session=session,+ )++ assert response is final+ assert session.request.call_count == 2+ assert all(+ call.kwargs.get("allow_redirects") is False+ for call in session.request.call_args_list+ )+++class TestWebIngestorSSRF:+ def test_private_ip_never_reaches_urllib3(self):+ ingestor = WebIngestor(respect_robots=False, delay=0)+ attempts = []++ def intercepting_urlopen(self, method, url, **kw):+ attempts.append((self.host, self.port, url))+ raise Exception("intercepted at urllib3.urlopen")++ urls = [+ "http://10.0.0.1/",+ "http://192.168.1.1/",+ "http://127.0.0.1:9999/internal",+ "http://169.254.169.254/latest/meta-data/",+ ]+ with patch.object(pool.HTTPConnectionPool, "urlopen", intercepting_urlopen):+ for url in urls:+ attempts.clear()+ with pytest.raises(ValidationError):+ ingestor.ingest_url(url)+ assert attempts == [], f"SSRF target reached urllib3: {url}"++ def test_file_scheme_rejected_by_semantica(self):+ ingestor = WebIngestor(respect_robots=False, delay=0)+ with pytest.raises(ValidationError, match="not permitted"):+ ingestor.ingest_url("file://localhost/etc/passwd")++ def test_allow_private_ips_permits_loopback_fetch(self):+ ingestor = WebIngestor(+ respect_robots=False, delay=0, allow_private_ips=True+ )+ with patch.object(ingestor.session, "request") as mock_request, patch.object(+ ingestor, "extract_content"+ ) as mock_extract:+ mock_resp = MagicMock()+ mock_resp.status_code = 200+ mock_resp.text = "ok"+ mock_resp.raise_for_status = MagicMock()+ mock_request.return_value = mock_resp+ mock_extract.return_value = MagicMock(status_code=None)++ ingestor.ingest_url("http://127.0.0.1:9/probe")++ mock_request.assert_called_once()+ assert mock_request.call_args.kwargs.get("allow_redirects") is False+ mock_extract.assert_called_once()++ def test_allow_private_ips_string_false_keeps_ssrf_on(self):+ ingestor = WebIngestor(+ respect_robots=False, delay=0, allow_private_ips="false"+ )+ assert ingestor.allow_private_ips is False+ with pytest.raises(ValidationError, match="blocked"):+ ingestor.ingest_url("http://127.0.0.1:9/probe")++ def test_allow_private_ips_string_true_opts_in(self):+ ingestor = WebIngestor(+ respect_robots=False, delay=0, allow_private_ips="true"+ )+ assert ingestor.allow_private_ips is True+ with patch.object(ingestor.session, "request") as mock_request, patch.object(+ ingestor, "extract_content"+ ) as mock_extract:+ mock_resp = MagicMock()+ mock_resp.status_code = 200+ mock_resp.text = "ok"+ mock_resp.raise_for_status = MagicMock()+ mock_request.return_value = mock_resp+ mock_extract.return_value = MagicMock(status_code=None)++ ingestor.ingest_url("http://127.0.0.1:9/probe")+ mock_request.assert_called_once()++ def test_redirect_to_private_ip_blocked(self):+ ingestor = WebIngestor(respect_robots=False, delay=0)+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "http://127.0.0.1/admin"}+ redirect.close = MagicMock()++ with patch.object(ingestor.session, "request", return_value=redirect), patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ with pytest.raises(ValidationError, match="blocked"):+ ingestor.ingest_url("https://example.com/public")+++class TestSitemapCrawlerSSRF:+ def test_private_sitemap_url_rejected(self):+ crawler = SitemapCrawler()+ with pytest.raises(ValidationError):+ crawler.parse_sitemap("http://10.0.0.1/sitemap.xml")++ def test_allow_private_ips_string_false_keeps_ssrf_on(self):+ crawler = SitemapCrawler(allow_private_ips="false")+ assert crawler.allow_private_ips is False+ with pytest.raises(ValidationError):+ crawler.parse_sitemap("http://10.0.0.1/sitemap.xml")++ def test_allow_private_ips_string_true_opts_in(self):+ crawler = SitemapCrawler(allow_private_ips="true")+ assert crawler.allow_private_ips is True++ def test_redirect_to_private_ip_blocked(self):+ crawler = SitemapCrawler()+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"}+ redirect.close = MagicMock()++ with patch(+ "semantica.ingest.ssrf.requests.request", return_value=redirect+ ), patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ with pytest.raises(ValidationError, match="blocked"):+ crawler.parse_sitemap("https://example.com/sitemap.xml")+++class TestRESTIngestorSSRF:+ def test_private_endpoint_never_reaches_session(self):+ with patch("requests.Session") as MockSession:+ mock_session = MockSession.return_value+ ingestor = RESTIngestor()+ with pytest.raises(ValidationError):+ ingestor.ingest_endpoint("http://169.254.169.254/latest/meta-data/")+ mock_session.request.assert_not_called()++ def test_file_scheme_rejected(self):+ ingestor = RESTIngestor()+ with pytest.raises(ValidationError, match="not permitted"):+ ingestor.ingest_endpoint("file://localhost/secret")++ def test_allow_private_ips_opt_in(self):+ with patch("requests.Session") as MockSession:+ mock_session = MockSession.return_value+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"ok": True}+ mock_response.headers = {"Content-Type": "application/json"}+ mock_session.request.return_value = mock_response++ ingestor = RESTIngestor(allow_private_ips=True)+ data = ingestor.ingest_endpoint("http://127.0.0.1:8080/health")+ assert data.data == {"ok": True}+ mock_session.request.assert_called_once()+ assert mock_session.request.call_args.kwargs.get("allow_redirects") is False++ def test_allow_private_ips_string_false_keeps_ssrf_on(self):+ with patch("requests.Session") as MockSession:+ mock_session = MockSession.return_value+ ingestor = RESTIngestor(allow_private_ips="false")+ assert ingestor.allow_private_ips is False+ with pytest.raises(ValidationError, match="blocked"):+ ingestor.ingest_endpoint("http://127.0.0.1:8080/health")+ mock_session.request.assert_not_called()++ def test_allow_private_ips_string_true_opts_in(self):+ with patch("requests.Session") as MockSession:+ mock_session = MockSession.return_value+ mock_response = MagicMock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"ok": True}+ mock_response.headers = {"Content-Type": "application/json"}+ mock_session.request.return_value = mock_response++ ingestor = RESTIngestor(allow_private_ips="true")+ assert ingestor.allow_private_ips is True+ data = ingestor.ingest_endpoint("http://127.0.0.1:8080/health")+ assert data.data == {"ok": True}+ mock_session.request.assert_called_once()++ def test_redirect_to_private_ip_blocked(self):+ with patch("requests.Session") as MockSession:+ mock_session = MockSession.return_value+ mock_session.headers = {}+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "http://127.0.0.1/secret"}+ redirect.close = MagicMock()+ mock_session.request.return_value = redirect++ ingestor = RESTIngestor()+ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ with pytest.raises(ValidationError, match="blocked"):+ ingestor.ingest_endpoint("https://example.com/api")+ mock_session.request.assert_called_once()