Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(response): Option to set the referer as the previous response url.

Karim shoairdd5c30c52122
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index 754d9f2..f5b8f62 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -68,6 +68,7 @@ class Response(Selector):
priority: int | None = None,
dont_filter: bool = False,
meta: dict[str, Any] | None = None,
+ referer_flow: bool = True,
**kwargs: Any,
) -> Any:
"""Create a Request to follow a URL.
@@ -82,6 +83,7 @@ class Response(Selector):
:param priority: The priority number to use, the higher the number, the higher priority to be processed first.
:param dont_filter: If this request has been done before, disable the filter to allow it again.
:param meta: Additional meta data to included in the request
+ :param referer_flow: Enabled by default, set the current response url as referer for the new request url.
:param kwargs: Additional Request arguments
:return: Request object ready to be yielded
"""
@@ -90,6 +92,22 @@ class Response(Selector):
if not self.request or not isinstance(self.request, Request):
raise TypeError("This response has no request set yet.")
+ # Merge original session kwargs with new kwargs (new takes precedence)
+ session_kwargs = {**self.request._session_kwargs, **kwargs}
+
+ if referer_flow:
+ # For requests
+ headers = session_kwargs.get("headers", {})
+ headers["referer"] = self.url
+ session_kwargs["headers"] = headers
+
+ # For browsers
+ extra_headers = session_kwargs.get("extra_headers", {})
+ extra_headers["referer"] = self.url
+ session_kwargs["extra_headers"] = extra_headers
+
+ session_kwargs["google_search"] = False
+
return Request(
url=self.urljoin(url),
sid=sid or self.request.sid,
@@ -97,7 +115,7 @@ class Response(Selector):
priority=priority if priority is not None else self.request.priority,
dont_filter=dont_filter,
meta={**(self.meta or {}), **(meta or {})},
- **(kwargs if kwargs else self.request._session_kwargs),
+ **session_kwargs,
)
def __str__(self) -> str: