Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(security): default follow_redirects to "safe" for SSRF protection

Karim shoaire7f9adb40a98
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.py
index d9a42fc..9778318 100644
--- a/scrapling/core/_shell_signatures.py
+++ b/scrapling/core/_shell_signatures.py
@@ -8,6 +8,7 @@ from scrapling.core._types import (
Optional,
SetCookieParam,
SelectorWaitStates,
+ FollowRedirects,
)
# Parameter definitions for shell function signatures (defined once at module level)
@@ -26,7 +27,7 @@ _REQUESTS_PARAMS = {
"headers": Any,
"retries": Optional[int],
"retry_delay": Optional[int],
- "follow_redirects": Optional[bool],
+ "follow_redirects": Optional[FollowRedirects],
"max_redirects": Optional[int],
"verify": Optional[bool],
"cert": Optional[str | Tuple[str, str]],
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index 25ddb5b..fc8a075 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]
PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]
extraction_types = Literal["text", "html", "markdown"]
StrOrBytes = Union[str, bytes]
+FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]]
# Copied from `playwright._impl._api_structures.SetCookieParam`
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 208f3e7..9b33821 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -27,6 +27,7 @@ from scrapling.core._types import (
SetCookieParam,
extraction_types,
SelectorWaitStates,
+ FollowRedirects,
)
SessionType = Literal["dynamic", "stealthy"]
@@ -262,7 +263,7 @@ class ScraplingMCPServer:
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Dict[str, str]] = None,
timeout: Optional[int | float] = 30,
- follow_redirects: bool = True,
+ follow_redirects: FollowRedirects = "safe",
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
@@ -289,7 +290,7 @@ class ScraplingMCPServer:
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
- :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -335,7 +336,7 @@ class ScraplingMCPServer:
headers: Optional[Mapping[str, Optional[str]]] = None,
cookies: Optional[Dict[str, str]] = None,
timeout: Optional[int | float] = 30,
- follow_redirects: bool = True,
+ follow_redirects: FollowRedirects = "safe",
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
@@ -362,7 +363,7 @@ class ScraplingMCPServer:
:param headers: Headers to include in the request.
:param cookies: Cookies to use in the request.
:param timeout: Number of seconds to wait before timing out.
- :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 14a25fe..cb805c6 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -294,7 +294,7 @@ class CurlParser:
headers=headers,
cookies=cookies,
proxy=proxies,
- follow_redirects=True, # Scrapling default is True
+ follow_redirects="safe", # Follows redirects but rejects those to internal/private IPs
)
def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:
diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.py
index 6ab40b9..2741407 100644
--- a/scrapling/engines/_browsers/_types.py
+++ b/scrapling/engines/_browsers/_types.py
@@ -19,6 +19,7 @@ from scrapling.core._types import (
TypeAlias,
SetCookieParam,
SelectorWaitStates,
+ FollowRedirects,
)
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
@@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False):
headers: Optional[Mapping[str, Optional[str]]]
retries: Optional[int]
retry_delay: Optional[int]
- follow_redirects: Optional[bool]
+ follow_redirects: Optional[FollowRedirects]
max_redirects: Optional[int]
verify: Optional[bool]
cert: Optional[str | Tuple[str, str]]
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index a962ccf..fae9f83 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -20,6 +20,7 @@ from scrapling.core._types import (
Optional,
Awaitable,
SUPPORTED_HTTP_METHODS,
+ FollowRedirects,
)
from .toolbelt.custom import Response
@@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC):
self._default_headers = kwargs.get("headers") or {}
self._default_retries = kwargs.get("retries", 3)
self._default_retry_delay = kwargs.get("retry_delay", 1)
- self._default_follow_redirects = kwargs.get("follow_redirects", True)
+ self._default_follow_redirects = kwargs.get("follow_redirects", "safe")
self._default_max_redirects = kwargs.get("max_redirects", 30)
self._default_verify = kwargs.get("verify", True)
self._default_cert = kwargs.get("cert") or None
@@ -285,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -317,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -349,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -381,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -502,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -534,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -566,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -598,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):
- headers: Headers to include in the request.
- cookies: Cookies to use in the request.
- timeout: Number of seconds to wait before timing out.
- - follow_redirects: Whether to follow redirects. Defaults to True.
+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).
- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
- retries: Number of retry attempts. Defaults to 3.
- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
@@ -663,7 +664,7 @@ class FetcherSession:
headers: Optional[Dict[str, str]] = None,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
- follow_redirects: bool = True,
+ follow_redirects: FollowRedirects = "safe",
max_redirects: int = 30,
verify: bool = True,
cert: Optional[str | Tuple[str, str]] = None,
@@ -682,7 +683,7 @@ class FetcherSession:
:param headers: Headers to include in the session with every request.
:param retries: Number of retry attempts. Defaults to 3.
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
- :param follow_redirects: Whether to follow redirects. Defaults to True.
+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.
:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.
:param verify: Whether to verify HTTPS certificates. Defaults to True.
:param cert: Tuple of (cert, key) filenames for the client certificate.