🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(security): default follow_redirects to "safe" for SSRF protection
Karim shoaire7f9adb40a98
diff --git a/scrapling/core/_shell_signatures.py b/scrapling/core/_shell_signatures.pyindex d9a42fc..9778318 100644--- a/scrapling/core/_shell_signatures.py+++ b/scrapling/core/_shell_signatures.py@@ -8,6 +8,7 @@ from scrapling.core._types import (Optional,SetCookieParam,SelectorWaitStates,+ FollowRedirects,)# Parameter definitions for shell function signatures (defined once at module level)@@ -26,7 +27,7 @@ _REQUESTS_PARAMS = {"headers": Any,"retries": Optional[int],"retry_delay": Optional[int],- "follow_redirects": Optional[bool],+ "follow_redirects": Optional[FollowRedirects],"max_redirects": Optional[int],"verify": Optional[bool],"cert": Optional[str | Tuple[str, str]],diff --git a/scrapling/core/_types.py b/scrapling/core/_types.pyindex 25ddb5b..fc8a075 100644--- a/scrapling/core/_types.py+++ b/scrapling/core/_types.py@@ -40,6 +40,7 @@ SelectorWaitStates = Literal["attached", "detached", "hidden", "visible"]PageLoadStates = Literal["commit", "domcontentloaded", "load", "networkidle"]extraction_types = Literal["text", "html", "markdown"]StrOrBytes = Union[str, bytes]+FollowRedirects = Union[bool, Literal["safe", "all", "obeycode", "firstonly"]]# Copied from `playwright._impl._api_structures.SetCookieParam`diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 208f3e7..9b33821 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -27,6 +27,7 @@ from scrapling.core._types import (SetCookieParam,extraction_types,SelectorWaitStates,+ FollowRedirects,)SessionType = Literal["dynamic", "stealthy"]@@ -262,7 +263,7 @@ class ScraplingMCPServer:headers: Optional[Mapping[str, Optional[str]]] = None,cookies: Optional[Dict[str, str]] = None,timeout: Optional[int | float] = 30,- follow_redirects: bool = True,+ follow_redirects: FollowRedirects = "safe",max_redirects: int = 30,retries: Optional[int] = 3,retry_delay: Optional[int] = 1,@@ -289,7 +290,7 @@ class ScraplingMCPServer::param headers: Headers to include in the request.:param cookies: Cookies to use in the request.:param timeout: Number of seconds to wait before timing out.- :param follow_redirects: Whether to follow redirects. Defaults to True.+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.:param retries: Number of retry attempts. Defaults to 3.:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -335,7 +336,7 @@ class ScraplingMCPServer:headers: Optional[Mapping[str, Optional[str]]] = None,cookies: Optional[Dict[str, str]] = None,timeout: Optional[int | float] = 30,- follow_redirects: bool = True,+ follow_redirects: FollowRedirects = "safe",max_redirects: int = 30,retries: Optional[int] = 3,retry_delay: Optional[int] = 1,@@ -362,7 +363,7 @@ class ScraplingMCPServer::param headers: Headers to include in the request.:param cookies: Cookies to use in the request.:param timeout: Number of seconds to wait before timing out.- :param follow_redirects: Whether to follow redirects. Defaults to True.+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.:param retries: Number of retry attempts. Defaults to 3.:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 14a25fe..cb805c6 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -294,7 +294,7 @@ class CurlParser:headers=headers,cookies=cookies,proxy=proxies,- follow_redirects=True, # Scrapling default is True+ follow_redirects="safe", # Follows redirects but rejects those to internal/private IPs)def convert2fetcher(self, curl_command: Request | str) -> Optional[Response]:diff --git a/scrapling/engines/_browsers/_types.py b/scrapling/engines/_browsers/_types.pyindex 6ab40b9..2741407 100644--- a/scrapling/engines/_browsers/_types.py+++ b/scrapling/engines/_browsers/_types.py@@ -19,6 +19,7 @@ from scrapling.core._types import (TypeAlias,SetCookieParam,SelectorWaitStates,+ FollowRedirects,)from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator@@ -39,7 +40,7 @@ class RequestsSession(TypedDict, total=False):headers: Optional[Mapping[str, Optional[str]]]retries: Optional[int]retry_delay: Optional[int]- follow_redirects: Optional[bool]+ follow_redirects: Optional[FollowRedirects]max_redirects: Optional[int]verify: Optional[bool]cert: Optional[str | Tuple[str, str]]diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex a962ccf..fae9f83 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -20,6 +20,7 @@ from scrapling.core._types import (Optional,Awaitable,SUPPORTED_HTTP_METHODS,+ FollowRedirects,)from .toolbelt.custom import Response@@ -77,7 +78,7 @@ class _ConfigurationLogic(ABC):self._default_headers = kwargs.get("headers") or {}self._default_retries = kwargs.get("retries", 3)self._default_retry_delay = kwargs.get("retry_delay", 1)- self._default_follow_redirects = kwargs.get("follow_redirects", True)+ self._default_follow_redirects = kwargs.get("follow_redirects", "safe")self._default_max_redirects = kwargs.get("max_redirects", 30)self._default_verify = kwargs.get("verify", True)self._default_cert = kwargs.get("cert") or None@@ -285,7 +286,7 @@ class _SyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -317,7 +318,7 @@ class _SyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -349,7 +350,7 @@ class _SyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -381,7 +382,7 @@ class _SyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -502,7 +503,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -534,7 +535,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -566,7 +567,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -598,7 +599,7 @@ class _ASyncSessionLogic(_ConfigurationLogic):- headers: Headers to include in the request.- cookies: Cookies to use in the request.- timeout: Number of seconds to wait before timing out.- - follow_redirects: Whether to follow redirects. Defaults to True.+ - follow_redirects: Whether to follow redirects. Defaults to "safe" (rejects redirects to internal/private IPs).- max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.- retries: Number of retry attempts. Defaults to 3.- retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -663,7 +664,7 @@ class FetcherSession:headers: Optional[Dict[str, str]] = None,retries: Optional[int] = 3,retry_delay: Optional[int] = 1,- follow_redirects: bool = True,+ follow_redirects: FollowRedirects = "safe",max_redirects: int = 30,verify: bool = True,cert: Optional[str | Tuple[str, str]] = None,@@ -682,7 +683,7 @@ class FetcherSession::param headers: Headers to include in the session with every request.:param retries: Number of retry attempts. Defaults to 3.:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.- :param follow_redirects: Whether to follow redirects. Defaults to True.+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.:param verify: Whether to verify HTTPS certificates. Defaults to True.:param cert: Tuple of (cert, key) filenames for the client certificate.