Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
feat(seed): allow_private_ips opt-in for trusted internal API sources (#959)
Yunare Maiac5d13a45db48
diff --git a/semantica/ingest/ssrf.py b/semantica/ingest/ssrf.pyindex ade93a46..083fbcca 100644--- a/semantica/ingest/ssrf.py+++ b/semantica/ingest/ssrf.py@@ -33,6 +33,46 @@ _DEFAULT_MAX_REDIRECTS = 10_REDIRECT_STATUS_CODES = frozenset({301, 302, 303, 307, 308})_STRIP_BODY_ON_REDIRECT = frozenset({301, 302, 303})+# Standard port per scheme (mirrors requests' DEFAULT_PORTS).+_DEFAULT_PORTS = {"http": 80, "https": 443}+++def _should_strip_auth(old_url: str, new_url: str) -> bool:+ """Decide whether credentials must not follow a redirect.++ Mirrors ``requests.utils.should_strip_auth``: credentials are stripped+ when the hostname changes, when the port changes (outside default+ ports), or on an https -> http downgrade on the same host. The single+ exception is an http -> https upgrade on default ports, which requests+ treats as safe to keep the credential for.+ """+ old_parsed = urlparse(old_url)+ new_parsed = urlparse(new_url)++ if old_parsed.hostname != new_parsed.hostname:+ return True++ # Special case: allow http -> https redirect on standard ports.+ if (+ old_parsed.scheme == "http"+ and old_parsed.port in (80, None)+ and new_parsed.scheme == "https"+ and new_parsed.port in (443, None)+ ):+ return False++ changed_port = old_parsed.port != new_parsed.port+ changed_scheme = old_parsed.scheme != new_parsed.scheme+ default_port = (_DEFAULT_PORTS.get(old_parsed.scheme), None)+ if (+ not changed_scheme+ and old_parsed.port in default_port+ and new_parsed.port in default_port+ ):+ return False++ return changed_port or changed_scheme+_dns_executor: Optional[concurrent.futures.ThreadPoolExecutor] = None_dns_executor_lock = threading.Lock()@@ -276,6 +316,16 @@ def request_with_ssrf_guard(next_url = urljoin(current_url, str(location).strip())validate_url_for_request(next_url, allow_private_ips=allow_private_ips)+ # Do not leak sensitive headers to a different origin on redirects:+ # reuse the caller's headers only while host, port, and scheme keep+ # the credential safe, mirroring requests' should_strip_auth.+ if _should_strip_auth(current_url, next_url):+ kwargs = dict(kwargs)+ headers = dict(kwargs.get("headers") or {})+ for sensitive in ("Authorization", "Proxy-Authorization"):+ headers.pop(sensitive, None)+ kwargs["headers"] = headers+# Match requests' historical method rewriting for 301/302/303.if (response.status_code in _STRIP_BODY_ON_REDIRECTdiff --git a/semantica/seed/seed_manager.py b/semantica/seed/seed_manager.pyindex 0d3bb046..5051697d 100644--- a/semantica/seed/seed_manager.py+++ b/semantica/seed/seed_manager.py@@ -43,6 +43,7 @@ from ..utils.helpers import read_json_file, write_json_filefrom ..utils.logging import get_loggerfrom ..utils.progress_tracker import get_progress_trackerfrom ..utils.types import EntityDict, RelationshipDict+from ..ingest.ssrf import parse_bool, request_with_ssrf_guard@dataclass@@ -453,6 +454,13 @@ class SeedDataManager:'entities', 'data', 'results', 'items' keys). Automatically addsentity_type, relationship_type, and source metadata if provided.+ SSRF protection is enabled by default: URLs resolving to private,+ loopback, link-local (including cloud metadata endpoints such as+ 169.254.169.254), or other blocked addresses are rejected, and every+ redirect hop is re-validated before being followed. For trusted+ internal deployments, pass ``allow_private_ips=True`` in the manager+ config to opt in (documented for internal use only).+Args:api_url: Base API URLendpoint: Optional API endpoint path (appended to api_url)@@ -491,8 +499,20 @@ class SeedDataManager:if api_key:request_headers["Authorization"] = f"Bearer {api_key}"- # Make API request- response = requests.get(full_url, headers=request_headers, timeout=30)+ # SSRF guard: reject private/loopback/link-local targets by default.+ # Trusted internal deployments can opt in via config+ # (allow_private_ips=True) — see issue #943.+ allow_private = parse_bool(self.config.get("allow_private_ips", False))++ # Make API request (request_with_ssrf_guard validates the URL and+ # every redirect before each hop)+ response = request_with_ssrf_guard(+ "GET",+ full_url,+ headers=request_headers,+ timeout=30,+ allow_private_ips=allow_private,+ )response.raise_for_status()# Parse responsediff --git a/tests/ingest/test_ssrf_protection.py b/tests/ingest/test_ssrf_protection.pyindex 2da84530..d7d3ab9b 100644--- a/tests/ingest/test_ssrf_protection.py+++ b/tests/ingest/test_ssrf_protection.py@@ -182,6 +182,128 @@ class TestRequestWithSsrfGuardRedirects:session=session,)+ def test_strips_authorization_on_cross_host_redirect(self):+ """Sensitive headers must not leak to a different redirect host."""+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "https://other-host.example/final"}+ redirect.close = MagicMock()++ final = MagicMock()+ final.status_code = 200+ final.headers = {}++ session = MagicMock()+ session.request.side_effect = [redirect, final]++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ request_with_ssrf_guard(+ "GET",+ "https://example.com/start",+ session=session,+ headers={"Authorization": "Bearer secret-token"},+ )++ assert session.request.call_count == 2+ second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})+ assert "Authorization" not in second_call_headers+ # The first hop still had the credential+ first_call_headers = session.request.call_args_list[0].kwargs.get("headers", {})+ assert first_call_headers.get("Authorization") == "Bearer secret-token"++ def test_keeps_authorization_on_same_host_redirect(self):+ """Same-host redirects keep the credential (requests semantics)."""+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "https://example.com/final"}+ redirect.close = MagicMock()++ final = MagicMock()+ final.status_code = 200+ final.headers = {}++ session = MagicMock()+ session.request.side_effect = [redirect, final]++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ request_with_ssrf_guard(+ "GET",+ "https://example.com/start",+ session=session,+ headers={"Authorization": "Bearer secret-token"},+ )++ assert session.request.call_count == 2+ second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})+ assert second_call_headers.get("Authorization") == "Bearer secret-token"++ def test_strips_authorization_on_scheme_downgrade(self):+ """Credentials must not follow an https -> http downgrade on the same host."""+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "http://example.com/final"}+ redirect.close = MagicMock()++ final = MagicMock()+ final.status_code = 200+ final.headers = {}++ session = MagicMock()+ session.request.side_effect = [redirect, final]++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ request_with_ssrf_guard(+ "GET",+ "https://example.com/start",+ session=session,+ headers={"Authorization": "Bearer secret-token"},+ )++ assert session.request.call_count == 2+ second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})+ assert "Authorization" not in second_call_headers+ # The first hop still had the credential+ first_call_headers = session.request.call_args_list[0].kwargs.get("headers", {})+ assert first_call_headers.get("Authorization") == "Bearer secret-token"++ def test_keeps_authorization_on_scheme_upgrade(self):+ """Credentials survive an http -> https upgrade on default ports (requests semantics)."""+ redirect = MagicMock()+ redirect.status_code = 302+ redirect.headers = {"Location": "https://example.com/final"}+ redirect.close = MagicMock()++ final = MagicMock()+ final.status_code = 200+ final.headers = {}++ session = MagicMock()+ session.request.side_effect = [redirect, final]++ with patch(+ "semantica.ingest.ssrf.socket.getaddrinfo",+ return_value=[(None, None, None, None, ("93.184.216.34", 0))],+ ):+ request_with_ssrf_guard(+ "GET",+ "http://example.com/start",+ session=session,+ headers={"Authorization": "Bearer secret-token"},+ )++ assert session.request.call_count == 2+ second_call_headers = session.request.call_args_list[1].kwargs.get("headers", {})+ assert second_call_headers.get("Authorization") == "Bearer secret-token"+def test_follows_safe_redirect(self):redirect = MagicMock()redirect.status_code = 302diff --git a/tests/test_seed_manager.py b/tests/test_seed_manager.pyindex 0f73b40d..31f966f1 100644--- a/tests/test_seed_manager.py+++ b/tests/test_seed_manager.py@@ -147,11 +147,11 @@ def test_load_from_database_import_error(seed_manager):seed_manager.load_from_database("sqlite:///:memory:", query="SELECT 1")assert "Database ingestion module not available" in str(excinfo.value)-@patch("requests.get")-def test_load_from_api(mock_get, seed_manager):+@patch("semantica.seed.seed_manager.request_with_ssrf_guard")+def test_load_from_api(mock_guard, seed_manager):mock_response = MagicMock()mock_response.json.return_value = {"results": [{"id": 1, "name": "Alice"}]}- mock_get.return_value = mock_response+ mock_guard.return_value = mock_responserecords = seed_manager.load_from_api(api_url="http://api.example.com",@@ -162,7 +162,31 @@ def test_load_from_api(mock_get, seed_manager):assert len(records) == 1assert records[0]["id"] == 1assert records[0]["entity_type"] == "User"- mock_get.assert_called_once()+ mock_guard.assert_called_once()++def test_load_from_api_blocks_private_by_default(seed_manager):+ with pytest.raises(ProcessingError) as excinfo:+ seed_manager.load_from_api(api_url="http://127.0.0.1:8000/secret")+ assert "blocked" in str(excinfo.value).lower() or "not allowed" in str(excinfo.value).lower()++@patch("semantica.seed.seed_manager.request_with_ssrf_guard")+def test_load_from_api_allows_private_when_configured(mock_guard, seed_manager):+ mock_response = MagicMock()+ mock_response.json.return_value = {"results": [{"id": 1, "name": "Alice"}]}+ mock_guard.return_value = mock_response++ manager = SeedDataManager(config={"allow_private_ips": True})+ records = manager.load_from_api(+ api_url="http://127.0.0.1:8000",+ endpoint="users",+ entity_type="User"+ )++ assert len(records) == 1+ mock_guard.assert_called_once()+ # The opt-in flag must reach the guard+ call_kwargs = mock_guard.call_args[1]+ assert call_kwargs["allow_private_ips"] is Truedef test_load_source(seed_manager, temp_data_dir):json_file = temp_data_dir / "source.json"