Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(ingest): SSRF protection for Web and API ingestors (#867) (#906)

pravit-ampab5c12f9afc6
diff --git a/semantica/ingest/api_ingestor.py b/semantica/ingest/api_ingestor.py
index 30677681..d1f7a860 100644
--- a/semantica/ingest/api_ingestor.py
+++ b/semantica/ingest/api_ingestor.py
@@ -38,6 +38,7 @@ except (ImportError, OSError):
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+from .ssrf import parse_bool, request_with_ssrf_guard
@dataclass
@@ -78,11 +79,16 @@ class RESTIngestor:
Args:
config: Optional REST API ingestion configuration dictionary
- **kwargs: Additional configuration parameters (merged into config)
+ **kwargs: Additional configuration parameters (merged into config).
+ Recognized keys include ``allow_private_ips`` (default False) to
+ opt into fetching private/loopback/link-local endpoints.
"""
self.logger = get_logger("api_ingestor")
self.config = config or {}
self.config.update(kwargs)
+ self.allow_private_ips = parse_bool(
+ self.config.get("allow_private_ips"), default=False
+ )
# Initialize session with retry strategy
self.session = requests.Session()
@@ -103,7 +109,10 @@ class RESTIngestor:
# Initialize progress tracker
self.progress_tracker = get_progress_tracker()
- self.logger.debug("REST API ingestor initialized")
+ self.logger.debug(
+ "REST API ingestor initialized (allow_private_ips=%s)",
+ self.allow_private_ips,
+ )
def ingest_endpoint(
self,
@@ -137,7 +146,7 @@ class RESTIngestor:
- metadata: Additional metadata
Raises:
- ValidationError: If endpoint is invalid
+ ValidationError: If endpoint is invalid or fails SSRF checks
ProcessingError: If request fails
"""
tracking_id = self.progress_tracker.start_tracking(
@@ -153,10 +162,12 @@ class RESTIngestor:
if headers:
request_headers.update(headers)
- # Make request
- response = self.session.request(
- method=method,
- url=endpoint,
+ # Make request (SSRF-safe; validates URL and each redirect hop)
+ response = request_with_ssrf_guard(
+ method,
+ endpoint,
+ session=self.session,
+ allow_private_ips=self.allow_private_ips,
headers=request_headers,
params=params,
data=data,
@@ -196,6 +207,11 @@ class RESTIngestor:
},
)
+ except ValidationError:
+ self.progress_tracker.stop_tracking(
+ tracking_id, status="failed", message=f"Invalid endpoint URL: {endpoint}"
+ )
+ raise
except requests.exceptions.RequestException as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
diff --git a/semantica/ingest/methods.py b/semantica/ingest/methods.py
index a3e3ba76..ac9775a2 100644
--- a/semantica/ingest/methods.py
+++ b/semantica/ingest/methods.py
@@ -553,6 +553,12 @@ def ingest_web(
# Get config
config = ingest_config.get_method_config("web")
config.update(kwargs)
+ if "allow_private_ips" in config:
+ from .ssrf import parse_bool
+
+ config["allow_private_ips"] = parse_bool(
+ config["allow_private_ips"], default=False
+ )
ingestor = WebIngestor(**config)
diff --git a/semantica/ingest/ssrf.py b/semantica/ingest/ssrf.py
new file mode 100644
index 00000000..ade93a46
--- /dev/null
+++ b/semantica/ingest/ssrf.py
@@ -0,0 +1,293 @@
+"""SSRF safeguards for ingest HTTP clients.
+
+Validates outbound request URLs before they reach ``requests`` / urllib3 so
+user-supplied targets cannot reach private, loopback, or link-local
+addresses (including cloud metadata endpoints).
+"""
+
+from __future__ import annotations
+
+import concurrent.futures
+import ipaddress
+import socket
+import threading
+from typing import Any, Iterable, Optional
+from urllib.parse import urljoin, urlparse
+
+import requests
+
+from ..utils.exceptions import ValidationError
+
+ALLOWED_URL_SCHEMES = frozenset({"http", "https"})
+
+_TRUE_STRINGS = frozenset({"1", "true", "yes", "on"})
+_FALSE_STRINGS = frozenset({"0", "false", "no", "off"})
+
+# Keep DNS resolution bounded so fake/unreachable hosts in tests and offline
+# environments cannot hang request validation indefinitely.
+_DNS_RESOLVE_TIMEOUT_SECONDS = 2.0
+_DNS_EXECUTOR_WORKERS = 4
+
+# Bound manual redirect following so open redirect chains cannot hang fetches.
+_DEFAULT_MAX_REDIRECTS = 10
+_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})
+_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303})
+
+_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
+_dns_executor_lock = threading.Lock()
+
+
+def parse_bool(value: Any, default: bool = False) -> bool:
+ """Parse a config value into a bool without truthy-string pitfalls.
+
+ ``bool('false')`` is ``True`` in Python, which would silently disable SSRF
+ protections when string-typed config reaches ingestors. This helper accepts
+ only explicit bools and a small allowlist of string/int forms.
+
+ Args:
+ value: Config value to interpret. ``None`` yields *default*.
+ default: Value returned when *value* is ``None``.
+
+ Raises:
+ ValidationError: If *value* is not a recognized boolean form.
+ """
+ if value is None:
+ return default
+ if isinstance(value, bool):
+ return value
+ if isinstance(value, (int, float)):
+ if value == 1:
+ return True
+ if value == 0:
+ return False
+ raise ValidationError(f"Invalid boolean value: {value!r}")
+ if isinstance(value, str):
+ normalized = value.strip().lower()
+ if normalized in _TRUE_STRINGS:
+ return True
+ if normalized in _FALSE_STRINGS:
+ return False
+ raise ValidationError(f"Invalid boolean value: {value!r}")
+ raise ValidationError(
+ f"Invalid boolean type: {type(value).__name__} ({value!r})"
+ )
+
+
+def _shutdown_executor(executor: concurrent.futures.ThreadPoolExecutor) -> None:
+ """Shut down *executor* without waiting for in-flight DNS lookups."""
+ try:
+ executor.shutdown(wait=False, cancel_futures=True)
+ except TypeError:
+ # cancel_futures was added in Python 3.9.
+ executor.shutdown(wait=False)
+
+
+def _get_dns_executor() -> concurrent.futures.ThreadPoolExecutor:
+ """Return a process-wide executor for bounded DNS lookups."""
+ global _dns_executor
+ with _dns_executor_lock:
+ if _dns_executor is None:
+ _dns_executor = concurrent.futures.ThreadPoolExecutor(
+ max_workers=_DNS_EXECUTOR_WORKERS,
+ thread_name_prefix="semantica-ssrf-dns",
+ )
+ return _dns_executor
+
+
+# Explicit blocked networks from issue #867, plus common non-routable ranges.
+BLOCKED_NETWORKS = (
+ ipaddress.ip_network("0.0.0.0/8"),
+ ipaddress.ip_network("10.0.0.0/8"),
+ ipaddress.ip_network("127.0.0.0/8"),
+ ipaddress.ip_network("169.254.0.0/16"), # link-local / cloud metadata
+ ipaddress.ip_network("172.16.0.0/12"),
+ ipaddress.ip_network("192.168.0.0/16"),
+ ipaddress.ip_network("::1/128"),
+ ipaddress.ip_network("fc00::/7"),
+ ipaddress.ip_network("fe80::/10"),
+)
+
+
+def _ip_is_blocked(addr: ipaddress._BaseAddress) -> bool:
+ if (
+ addr.is_private
+ or addr.is_loopback
+ or addr.is_link_local
+ or addr.is_reserved
+ or addr.is_multicast
+ or addr.is_unspecified
+ ):
+ return True
+ return any(addr in network for network in BLOCKED_NETWORKS)
+
+
+def _hostname_resolves_to_blocked(hostname: str) -> bool:
+ """Return True if any resolved address for *hostname* is blocked.
+
+ DNS lookups run on a shared thread pool with ``Future.result(timeout=...)``.
+ Do not use ``with ThreadPoolExecutor(...)`` here: on timeout, leaving the
+ context waits for the hung ``getaddrinfo`` worker and defeats the bound.
+
+ Raises:
+ ValidationError: If DNS resolution fails or times out. Fail closed so
+ outbound requests never proceed without confirmed safe IPs.
+ """
+ executor = _get_dns_executor()
+ owned_executor = False
+ try:
+ try:
+ future = executor.submit(socket.getaddrinfo, hostname, None)
+ except RuntimeError:
+ # Shared executor was shut down; use a throwaway pool that never
+ # blocks the caller on shutdown.
+ executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
+ owned_executor = True
+ future = executor.submit(socket.getaddrinfo, hostname, None)
+ resolved: Iterable = future.result(timeout=_DNS_RESOLVE_TIMEOUT_SECONDS)
+ except (socket.gaierror, concurrent.futures.TimeoutError, OSError) as exc:
+ raise ValidationError(
+ f"URL host '{hostname}' could not be resolved safely "
+ "(DNS error or timeout); request blocked"
+ ) from exc
+ finally:
+ if owned_executor:
+ _shutdown_executor(executor)
+
+ for info in resolved:
+ sockaddr = info[4]
+ addr = ipaddress.ip_address(sockaddr[0])
+ if _ip_is_blocked(addr):
+ return True
+ return False
+
+
+def validate_url_for_request(
+ url: str, *, allow_private_ips: bool = False
+) -> None:
+ """Validate that *url* is safe to fetch over HTTP(S).
+
+ Args:
+ url: Absolute URL to validate.
+ allow_private_ips: When True, skip private/loopback/link-local checks
+ (for trusted internal deployments).
+
+ Raises:
+ ValidationError: If the scheme is not http/https, the URL is malformed,
+ or the host targets a blocked address space.
+ """
+ if not isinstance(url, str) or not url.strip():
+ raise ValidationError("URL must be a non-empty string")
+
+ parsed = urlparse(url.strip())
+ scheme = (parsed.scheme or "").lower()
+ if scheme not in ALLOWED_URL_SCHEMES:
+ raise ValidationError(
+ f"URL scheme '{parsed.scheme}' is not permitted. "
+ "Only http and https are allowed."
+ )
+ if not parsed.netloc:
+ raise ValidationError(
+ f"Invalid URL format: {url}. "
+ "URL must include scheme (http/https) and netloc (domain)."
+ )
+
+ host = parsed.hostname
+ if not host:
+ raise ValidationError(
+ f"Invalid URL format: {url}. "
+ "URL must include a hostname."
+ )
+
+ if allow_private_ips:
+ return
+
+ lowered = host.lower().rstrip(".")
+ if lowered == "localhost" or lowered.endswith(".localhost"):
+ raise ValidationError(f"URL host is not allowed: {host}")
+
+ try:
+ literal_ip = ipaddress.ip_address(host)
+ except ValueError:
+ literal_ip = None
+
+ if literal_ip is not None:
+ if _ip_is_blocked(literal_ip):
+ raise ValidationError(f"URL points to a blocked address: {host}")
+ return
+
+ if _hostname_resolves_to_blocked(host):
+ raise ValidationError(
+ f"URL host '{host}' resolves to a blocked (private/loopback/"
+ "link-local) address"
+ )
+
+
+def request_with_ssrf_guard(
+ method: str,
+ url: str,
+ *,
+ session: Optional[requests.Session] = None,
+ allow_private_ips: bool = False,
+ max_redirects: int = _DEFAULT_MAX_REDIRECTS,
+ **kwargs: Any,
+) -> requests.Response:
+ """Perform an HTTP request with SSRF checks on *url* and every redirect.
+
+ ``requests`` follows redirects by default, which would allow a validated
+ public URL to bounce into private/loopback/link-local space. This helper
+ disables automatic redirects and re-validates each ``Location`` target
+ before issuing the next hop.
+ """
+ kwargs = dict(kwargs)
+ kwargs.pop("allow_redirects", None)
+
+ validate_url_for_request(url, allow_private_ips=allow_private_ips)
+
+ requester = session.request if session is not None else requests.request
+ current_url = url
+ current_method = method.upper()
+ redirects_followed = 0
+
+ while True:
+ response = requester(
+ current_method,
+ current_url,
+ allow_redirects=False,
+ **kwargs,
+ )
+
+ if response.status_code not in _REDIRECT_STATUS_CODES:
+ return response
+
+ if redirects_followed >= max_redirects:
+ response.close()
+ raise ValidationError(
+ f"Exceeded maximum redirects ({max_redirects}) while "
+ f"fetching '{url}'"
+ )
+
+ location = response.headers.get("Location")
+ if not location or not str(location).strip():
+ response.close()
+ raise ValidationError(
+ f"Redirect from '{current_url}' is missing a Location header"
+ )
+
+ next_url = urljoin(current_url, str(location).strip())
+ validate_url_for_request(next_url, allow_private_ips=allow_private_ips)
+
+ # Match requests' historical method rewriting for 301/302/303.
+ if (
+ response.status_code in _STRIP_BODY_ON_REDIRECT
+ and current_method not in {"GET", "HEAD"}
+ ):
+ current_method = "GET"
+ for key in ("data", "json", "files"):
+ kwargs.pop(key, None)
+
+ # Params apply to the original request URL only; Location is authoritative.
+ kwargs.pop("params", None)
+
+ response.close()
+ current_url = next_url
+ redirects_followed += 1
diff --git a/semantica/ingest/web_ingestor.py b/semantica/ingest/web_ingestor.py
index e542ee39..5dd77f31 100644
--- a/semantica/ingest/web_ingestor.py
+++ b/semantica/ingest/web_ingestor.py
@@ -48,6 +48,7 @@ from urllib3.util.retry import Retry
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+from .ssrf import parse_bool, request_with_ssrf_guard, validate_url_for_request
@dataclass
@@ -338,10 +339,14 @@ class SitemapCrawler:
Sets up the crawler with configuration options.
Args:
- **config: Crawler configuration options (currently unused)
+ **config: Crawler configuration options. Recognized keys:
+ - allow_private_ips: Allow private/loopback sitemap hosts
"""
self.logger = get_logger("sitemap_crawler")
self.config = config
+ self.allow_private_ips = parse_bool(
+ config.get("allow_private_ips"), default=False
+ )
def parse_sitemap(self, sitemap_url: str) -> List[str]:
"""
@@ -359,10 +364,16 @@ class SitemapCrawler:
Raises:
ProcessingError: If sitemap cannot be fetched or parsed
+ ValidationError: If sitemap_url fails SSRF checks
"""
try:
- # Fetch sitemap
- response = requests.get(sitemap_url, timeout=30)
+ # Fetch sitemap (SSRF-safe; validates URL and each redirect hop)
+ response = request_with_ssrf_guard(
+ "GET",
+ sitemap_url,
+ allow_private_ips=self.allow_private_ips,
+ timeout=30,
+ )
response.raise_for_status()
# Parse XML
@@ -391,6 +402,8 @@ class SitemapCrawler:
)
return urls
+ except ValidationError:
+ raise
except Exception as e:
self.logger.error(f"Failed to parse sitemap {sitemap_url}: {e}")
raise ProcessingError(f"Failed to parse sitemap: {e}") from e
@@ -411,10 +424,16 @@ class SitemapCrawler:
Raises:
ProcessingError: If sitemap index cannot be fetched or parsed
+ ValidationError: If index_url fails SSRF checks
"""
try:
- # Fetch sitemap index
- response = requests.get(index_url, timeout=30)
+ # Fetch sitemap index (SSRF-safe; validates URL and each redirect hop)
+ response = request_with_ssrf_guard(
+ "GET",
+ index_url,
+ allow_private_ips=self.allow_private_ips,
+ timeout=30,
+ )
response.raise_for_status()
# Parse XML
@@ -448,6 +467,8 @@ class SitemapCrawler:
)
return all_urls
+ except ValidationError:
+ raise
except Exception as e:
self.logger.error(f"Failed to crawl sitemap index {index_url}: {e}")
raise ProcessingError(f"Failed to crawl sitemap index: {e}") from e
@@ -487,6 +508,7 @@ class WebIngestor:
max_retries: int = 3,
backoff_factor: float = 1.0,
timeout: int = 30,
+ allow_private_ips: bool = False,
config: Optional[Dict[str, Any]] = None,
**kwargs,
):
@@ -503,12 +525,19 @@ class WebIngestor:
max_retries: Maximum number of retry attempts (default: 3)
backoff_factor: Backoff factor for retries (default: 1.0)
timeout: Request timeout in seconds (default: 30)
+ allow_private_ips: Allow fetching private/loopback/link-local hosts
+ (default: False). Opt in only for trusted internal deployments.
config: Optional configuration dictionary (merged with kwargs)
**kwargs: Additional configuration parameters
"""
self.logger = get_logger("web_ingestor")
self.config = config or {}
self.config.update(kwargs)
+ self.allow_private_ips = parse_bool(
+ self.config.get("allow_private_ips", allow_private_ips),
+ default=False,
+ )
+ self.config["allow_private_ips"] = self.allow_private_ips
# Initialize HTTP session with retry strategy
self.session = requests.Session()
@@ -544,7 +573,8 @@ class WebIngestor:
self.logger.debug(
f"Web ingestor initialized: user_agent={user_agent}, "
- f"delay={delay}, respect_robots={respect_robots}"
+ f"delay={delay}, respect_robots={respect_robots}, "
+ f"allow_private_ips={self.allow_private_ips}"
)
def ingest_url(
@@ -574,16 +604,9 @@ class WebIngestor:
)
try:
- # Validate URL format
- try:
- parsed = urlparse(url)
- if not parsed.scheme or not parsed.netloc:
- raise ValidationError(
- f"Invalid URL format: {url}. "
- "URL must include scheme (http/https) and netloc (domain)."
- )
- except Exception as e:
- raise ValidationError(f"Invalid URL: {url}") from e
+ # Validate before robots check: RobotsChecker.read() makes an unguarded
+ # HTTP request to <host>/robots.txt and must not reach blocked addresses.
+ validate_url_for_request(url, allow_private_ips=self.allow_private_ips)
# Check robots.txt compliance
if self.robots_checker and not self.robots_checker.can_fetch(url):
@@ -593,11 +616,19 @@ class WebIngestor:
# Apply rate limiting (wait if necessary)
self.rate_limiter.wait_if_needed()
- # Fetch content with retry logic
+ # Fetch content with retry logic (SSRF-safe redirects)
try:
request_timeout = timeout or self.config.get("timeout", 30)
- response = self.session.get(url, timeout=request_timeout)
+ response = request_with_ssrf_guard(
+ "GET",
+ url,
+ session=self.session,
+ allow_private_ips=self.allow_private_ips,
+ timeout=request_timeout,
+ )
response.raise_for_status()
+ except ValidationError:
+ raise
except requests.RequestException as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
diff --git a/tests/ingest/test_ssrf_protection.py b/tests/ingest/test_ssrf_protection.py
new file mode 100644
index 00000000..2da84530
--- /dev/null
+++ b/tests/ingest/test_ssrf_protection.py
@@ -0,0 +1,407 @@
+"""SSRF protection tests for web and API ingestors (issue #867)."""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+import urllib3.connectionpool as pool
+
+from semantica.ingest.api_ingestor import RESTIngestor
+from semantica.ingest.ssrf import (
+ parse_bool,
+ request_with_ssrf_guard,
+ validate_url_for_request,
+)
+from semantica.ingest.web_ingestor import SitemapCrawler, WebIngestor
+from semantica.utils.exceptions import ValidationError
+
+
+class TestParseBool:
+ def test_bool_passthrough(self):
+ assert parse_bool(True) is True
+ assert parse_bool(False) is False
+
+ def test_none_uses_default(self):
+ assert parse_bool(None) is False
+ assert parse_bool(None, default=True) is True
+
+ def test_string_true_values(self):
+ for value in ("true", "TRUE", "1", "yes", "on", " Yes "):
+ assert parse_bool(value) is True
+
+ def test_string_false_values(self):
+ for value in ("false", "FALSE", "0", "no", "off", " No "):
+ assert parse_bool(value) is False
+
+ def test_int_zero_one(self):
+ assert parse_bool(0) is False
+ assert parse_bool(1) is True
+
+ def test_rejects_unknown_string(self):
+ with pytest.raises(ValidationError, match="Invalid boolean"):
+ parse_bool("maybe")
+
+
+class TestValidateUrlForRequest:
+ def test_accepts_https(self):
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ validate_url_for_request("https://example.com/path")
+
+ def test_rejects_file_scheme(self):
+ with pytest.raises(ValidationError, match="not permitted"):
+ validate_url_for_request("file://localhost/etc/passwd")
+
+ def test_rejects_gopher_scheme(self):
+ with pytest.raises(ValidationError, match="not permitted"):
+ validate_url_for_request("gopher://example.com/1")
+
+ def test_rejects_literal_private_ips(self):
+ for url in (
+ "http://10.0.0.1/",
+ "http://192.168.1.1/",
+ "http://172.16.5.5/internal",
+ "http://127.0.0.1:9999/internal",
+ "http://169.254.169.254/latest/meta-data/",
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ validate_url_for_request(url)
+
+ def test_rejects_localhost_hostname(self):
+ with pytest.raises(ValidationError, match="not allowed"):
+ validate_url_for_request("http://localhost/admin")
+
+ def test_rejects_hostname_resolving_to_private_ip(self):
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("10.0.0.5", 0))],
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ validate_url_for_request("http://internal.corp/secret")
+
+ def test_allow_private_ips_opt_in(self):
+ validate_url_for_request(
+ "http://127.0.0.1:8080/health", allow_private_ips=True
+ )
+ # Scheme allowlist still applies when private IPs are permitted
+ with pytest.raises(ValidationError, match="not permitted"):
+ validate_url_for_request(
+ "file://localhost/x", allow_private_ips=True
+ )
+
+ def test_dns_failure_raises(self):
+ import socket
+
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ side_effect=socket.gaierror("name or service not known"),
+ ):
+ with pytest.raises(ValidationError, match="could not be resolved safely"):
+ validate_url_for_request("http://does-not-resolve.invalid/path")
+
+ def test_dns_timeout_raises(self):
+ import concurrent.futures
+
+ with patch(
+ "semantica.ingest.ssrf.concurrent.futures.Future.result",
+ side_effect=concurrent.futures.TimeoutError(),
+ ):
+ with pytest.raises(ValidationError, match="could not be resolved safely"):
+ validate_url_for_request("http://slow-dns.example/path")
+
+ def test_hung_getaddrinfo_returns_within_timeout_bound(self):
+ """Timeout must not wait on executor shutdown for a blocking resolver."""
+ import time
+
+ import semantica.ingest.ssrf as ssrf
+
+ hang_seconds = 1.5
+ bound = 0.1
+
+ def hanging_getaddrinfo(*_args, **_kwargs):
+ time.sleep(hang_seconds)
+ return [(None, None, None, None, ("93.184.216.34", 0))]
+
+ with patch.object(ssrf, "_DNS_RESOLVE_TIMEOUT_SECONDS", bound), patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ side_effect=hanging_getaddrinfo,
+ ):
+ start = time.monotonic()
+ with pytest.raises(ValidationError, match="could not be resolved safely"):
+ ssrf._hostname_resolves_to_blocked("hanging.example")
+ elapsed = time.monotonic() - start
+
+ # Must fail closed near the configured bound, not after hang_seconds
+ # (which is what ``with ThreadPoolExecutor`` shutdown waiting causes).
+ assert elapsed < hang_seconds / 2
+ assert elapsed < bound + 0.75
+
+
+class TestRequestWithSsrfGuardRedirects:
+ def test_blocks_redirect_to_loopback(self):
+ redirect = MagicMock()
+ redirect.status_code = 302
+ redirect.headers = {"Location": "http://127.0.0.1/secret"}
+ redirect.close = MagicMock()
+
+ session = MagicMock()
+ session.request.return_value = redirect
+
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ request_with_ssrf_guard(
+ "GET",
+ "https://example.com/start",
+ session=session,
+ )
+
+ session.request.assert_called_once()
+ assert session.request.call_args.kwargs.get("allow_redirects") is False
+
+ def test_blocks_redirect_to_metadata_ip(self):
+ redirect = MagicMock()
+ redirect.status_code = 301
+ redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"}
+ redirect.close = MagicMock()
+
+ session = MagicMock()
+ session.request.return_value = redirect
+
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ request_with_ssrf_guard(
+ "GET",
+ "https://example.com/start",
+ session=session,
+ )
+
+ def test_follows_safe_redirect(self):
+ redirect = MagicMock()
+ redirect.status_code = 302
+ redirect.headers = {"Location": "https://example.com/final"}
+ redirect.close = MagicMock()
+
+ final = MagicMock()
+ final.status_code = 200
+ final.headers = {}
+
+ session = MagicMock()
+ session.request.side_effect = [redirect, final]
+
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ response = request_with_ssrf_guard(
+ "GET",
+ "https://example.com/start",
+ session=session,
+ )
+
+ assert response is final
+ assert session.request.call_count == 2
+ assert all(
+ call.kwargs.get("allow_redirects") is False
+ for call in session.request.call_args_list
+ )
+
+
+class TestWebIngestorSSRF:
+ def test_private_ip_never_reaches_urllib3(self):
+ ingestor = WebIngestor(respect_robots=False, delay=0)
+ attempts = []
+
+ def intercepting_urlopen(self, method, url, **kw):
+ attempts.append((self.host, self.port, url))
+ raise Exception("intercepted at urllib3.urlopen")
+
+ urls = [
+ "http://10.0.0.1/",
+ "http://192.168.1.1/",
+ "http://127.0.0.1:9999/internal",
+ "http://169.254.169.254/latest/meta-data/",
+ ]
+ with patch.object(pool.HTTPConnectionPool, "urlopen", intercepting_urlopen):
+ for url in urls:
+ attempts.clear()
+ with pytest.raises(ValidationError):
+ ingestor.ingest_url(url)
+ assert attempts == [], f"SSRF target reached urllib3: {url}"
+
+ def test_file_scheme_rejected_by_semantica(self):
+ ingestor = WebIngestor(respect_robots=False, delay=0)
+ with pytest.raises(ValidationError, match="not permitted"):
+ ingestor.ingest_url("file://localhost/etc/passwd")
+
+ def test_allow_private_ips_permits_loopback_fetch(self):
+ ingestor = WebIngestor(
+ respect_robots=False, delay=0, allow_private_ips=True
+ )
+ with patch.object(ingestor.session, "request") as mock_request, patch.object(
+ ingestor, "extract_content"
+ ) as mock_extract:
+ mock_resp = MagicMock()
+ mock_resp.status_code = 200
+ mock_resp.text = "ok"
+ mock_resp.raise_for_status = MagicMock()
+ mock_request.return_value = mock_resp
+ mock_extract.return_value = MagicMock(status_code=None)
+
+ ingestor.ingest_url("http://127.0.0.1:9/probe")
+
+ mock_request.assert_called_once()
+ assert mock_request.call_args.kwargs.get("allow_redirects") is False
+ mock_extract.assert_called_once()
+
+ def test_allow_private_ips_string_false_keeps_ssrf_on(self):
+ ingestor = WebIngestor(
+ respect_robots=False, delay=0, allow_private_ips="false"
+ )
+ assert ingestor.allow_private_ips is False
+ with pytest.raises(ValidationError, match="blocked"):
+ ingestor.ingest_url("http://127.0.0.1:9/probe")
+
+ def test_allow_private_ips_string_true_opts_in(self):
+ ingestor = WebIngestor(
+ respect_robots=False, delay=0, allow_private_ips="true"
+ )
+ assert ingestor.allow_private_ips is True
+ with patch.object(ingestor.session, "request") as mock_request, patch.object(
+ ingestor, "extract_content"
+ ) as mock_extract:
+ mock_resp = MagicMock()
+ mock_resp.status_code = 200
+ mock_resp.text = "ok"
+ mock_resp.raise_for_status = MagicMock()
+ mock_request.return_value = mock_resp
+ mock_extract.return_value = MagicMock(status_code=None)
+
+ ingestor.ingest_url("http://127.0.0.1:9/probe")
+ mock_request.assert_called_once()
+
+ def test_redirect_to_private_ip_blocked(self):
+ ingestor = WebIngestor(respect_robots=False, delay=0)
+ redirect = MagicMock()
+ redirect.status_code = 302
+ redirect.headers = {"Location": "http://127.0.0.1/admin"}
+ redirect.close = MagicMock()
+
+ with patch.object(ingestor.session, "request", return_value=redirect), patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ ingestor.ingest_url("https://example.com/public")
+
+
+class TestSitemapCrawlerSSRF:
+ def test_private_sitemap_url_rejected(self):
+ crawler = SitemapCrawler()
+ with pytest.raises(ValidationError):
+ crawler.parse_sitemap("http://10.0.0.1/sitemap.xml")
+
+ def test_allow_private_ips_string_false_keeps_ssrf_on(self):
+ crawler = SitemapCrawler(allow_private_ips="false")
+ assert crawler.allow_private_ips is False
+ with pytest.raises(ValidationError):
+ crawler.parse_sitemap("http://10.0.0.1/sitemap.xml")
+
+ def test_allow_private_ips_string_true_opts_in(self):
+ crawler = SitemapCrawler(allow_private_ips="true")
+ assert crawler.allow_private_ips is True
+
+ def test_redirect_to_private_ip_blocked(self):
+ crawler = SitemapCrawler()
+ redirect = MagicMock()
+ redirect.status_code = 302
+ redirect.headers = {"Location": "http://169.254.169.254/latest/meta-data/"}
+ redirect.close = MagicMock()
+
+ with patch(
+ "semantica.ingest.ssrf.requests.request", return_value=redirect
+ ), patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ crawler.parse_sitemap("https://example.com/sitemap.xml")
+
+
+class TestRESTIngestorSSRF:
+ def test_private_endpoint_never_reaches_session(self):
+ with patch("requests.Session") as MockSession:
+ mock_session = MockSession.return_value
+ ingestor = RESTIngestor()
+ with pytest.raises(ValidationError):
+ ingestor.ingest_endpoint("http://169.254.169.254/latest/meta-data/")
+ mock_session.request.assert_not_called()
+
+ def test_file_scheme_rejected(self):
+ ingestor = RESTIngestor()
+ with pytest.raises(ValidationError, match="not permitted"):
+ ingestor.ingest_endpoint("file://localhost/secret")
+
+ def test_allow_private_ips_opt_in(self):
+ with patch("requests.Session") as MockSession:
+ mock_session = MockSession.return_value
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"ok": True}
+ mock_response.headers = {"Content-Type": "application/json"}
+ mock_session.request.return_value = mock_response
+
+ ingestor = RESTIngestor(allow_private_ips=True)
+ data = ingestor.ingest_endpoint("http://127.0.0.1:8080/health")
+ assert data.data == {"ok": True}
+ mock_session.request.assert_called_once()
+ assert mock_session.request.call_args.kwargs.get("allow_redirects") is False
+
+ def test_allow_private_ips_string_false_keeps_ssrf_on(self):
+ with patch("requests.Session") as MockSession:
+ mock_session = MockSession.return_value
+ ingestor = RESTIngestor(allow_private_ips="false")
+ assert ingestor.allow_private_ips is False
+ with pytest.raises(ValidationError, match="blocked"):
+ ingestor.ingest_endpoint("http://127.0.0.1:8080/health")
+ mock_session.request.assert_not_called()
+
+ def test_allow_private_ips_string_true_opts_in(self):
+ with patch("requests.Session") as MockSession:
+ mock_session = MockSession.return_value
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.json.return_value = {"ok": True}
+ mock_response.headers = {"Content-Type": "application/json"}
+ mock_session.request.return_value = mock_response
+
+ ingestor = RESTIngestor(allow_private_ips="true")
+ assert ingestor.allow_private_ips is True
+ data = ingestor.ingest_endpoint("http://127.0.0.1:8080/health")
+ assert data.data == {"ok": True}
+ mock_session.request.assert_called_once()
+
+ def test_redirect_to_private_ip_blocked(self):
+ with patch("requests.Session") as MockSession:
+ mock_session = MockSession.return_value
+ mock_session.headers = {}
+ redirect = MagicMock()
+ redirect.status_code = 302
+ redirect.headers = {"Location": "http://127.0.0.1/secret"}
+ redirect.close = MagicMock()
+ mock_session.request.return_value = redirect
+
+ ingestor = RESTIngestor()
+ with patch(
+ "semantica.ingest.ssrf.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],
+ ):
+ with pytest.raises(ValidationError, match="blocked"):
+ ingestor.ingest_endpoint("https://example.com/api")
+ mock_session.request.assert_called_once()