Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(spiders): Add follow function to the response

Karim shoair0634f5796cd3
diff --git a/scrapling/core/_types.py b/scrapling/core/_types.py
index d7c1f8b..c897954 100644
--- a/scrapling/core/_types.py
+++ b/scrapling/core/_types.py
@@ -12,6 +12,7 @@ from typing import (
Callable,
Dict,
Generator,
+ AsyncGenerator,
Generic,
Iterable,
List,
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index d1f95b7..754d9f2 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -10,12 +10,19 @@ from scrapling.core._types import (
Dict,
cast,
List,
- Optional,
Tuple,
+ Union,
+ Optional,
+ Callable,
+ TYPE_CHECKING,
+ AsyncGenerator,
)
from scrapling.core.custom_types import MappingProxyType
from scrapling.parser import Selector, SQLiteStorageSystem
+if TYPE_CHECKING:
+ from scrapling.spiders import Request
+
class Response(Selector):
"""This class is returned by all engines as a way to unify the response type between different libraries."""
@@ -50,6 +57,52 @@ class Response(Selector):
# For easier debugging while working from a Python shell
log.info(f"Fetched ({status}) <{method} {url}> (referer: {request_headers.get('referer')})")
+ self.meta: Dict[str, Any] = {}
+ self.request: Optional["Request"] = None # Will be set by crawler
+
+ def follow(
+ self,
+ url: str,
+ sid: str = "",
+ callback: Callable[["Response"], AsyncGenerator[Union[Dict[str, Any], "Request", None], None]] | None = None,
+ priority: int | None = None,
+ dont_filter: bool = False,
+ meta: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> Any:
+ """Create a Request to follow a URL.
+
+ This is a helper method for spiders to easily follow links found in pages.
+
+ **IMPORTANT**: The below arguments if left empty, the corresponding value from the previous request will be used. The only exception is `dont_filter`.
+
+ :param url: The URL to follow (can be relative, will be joined with current URL)
+ :param sid: The session id to use
+ :param callback: Spider callback method to use
+ :param priority: The priority number to use, the higher the number, the higher priority to be processed first.
+ :param dont_filter: If this request has been done before, disable the filter to allow it again.
+ :param meta: Additional meta data to included in the request
+ :param kwargs: Additional Request arguments
+ :return: Request object ready to be yielded
+ """
+ from scrapling.spiders import Request
+
+ if not self.request or not isinstance(self.request, Request):
+ raise TypeError("This response has no request set yet.")
+
+ return Request(
+ url=self.urljoin(url),
+ sid=sid or self.request.sid,
+ callback=callback or self.request.callback,
+ priority=priority if priority is not None else self.request.priority,
+ dont_filter=dont_filter,
+ meta={**(self.meta or {}), **(meta or {})},
+ **(kwargs if kwargs else self.request._session_kwargs),
+ )
+
+ def __str__(self) -> str:
+ return f"<{self.status} {self.url}>"
+
class BaseFetcher:
__slots__ = ()