🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(Fetcher): Add the redirections history to Fetcher
Karim shoair3d2418b67a5d
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 9d5bed7..a939c46 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -72,6 +72,7 @@ class StaticEngine:headers=dict(response.headers),request_headers=dict(response.request.headers),method=response.request.method,+ history=[self._prepare_response(redirection) for redirection in response.history],**self.adaptor_arguments)diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex 9705eb3..dd31f4f 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -85,13 +85,14 @@ class Response(Adaptor):"""This class is returned by all engines as a way to unify response type between different libraries."""def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,- encoding: str = 'utf-8', method: str = 'GET', **adaptor_arguments: Dict):+ encoding: str = 'utf-8', method: str = 'GET', history: List = None, **adaptor_arguments: Dict):automatch_domain = adaptor_arguments.pop('automatch_domain', None)self.status = statusself.reason = reasonself.cookies = cookiesself.headers = headersself.request_headers = request_headers+ self.history = history or []encoding = ResponseEncoding.get_value(encoding, text)super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)# For back-ward compatibilitydiff --git a/scrapling/parser.py b/scrapling/parser.pyindex e6b138a..b83d867 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -132,7 +132,7 @@ class Adaptor(SelectorsGeneration):self.__tag = None# No need to check if all response attributes exist or not because if `status` exist, then the rest exist (Save some CPU cycles for speed)self.__response_data = {- key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'headers', 'request_headers',)+ key: getattr(self, key) for key in ('status', 'reason', 'cookies', 'history', 'headers', 'request_headers',)} if hasattr(self, 'status') else {}# Node functionalities, I wanted to move to separate Mixin class but it had slight impact on performance