🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: logging for response status
Karim shoair9f0001af26a8
diff --git a/scrapling/core/utils.py b/scrapling/core/utils.pyindex 35f8d0a..a911757 100644--- a/scrapling/core/utils.py+++ b/scrapling/core/utils.py@@ -14,8 +14,9 @@ from functools import lru_cache as cache # isort:skiphtml_forbidden = {html.HtmlComment, }logging.basicConfig(- level=logging.ERROR,- format='%(asctime)s - %(levelname)s - %(message)s',+ level=logging.INFO,+ format="[%(asctime)s] %(levelname)s: %(message)s",+ datefmt="%Y-%m-%d %H:%M:%S",handlers=[logging.StreamHandler()]diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex eabd219..7c8acd6 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -85,7 +85,8 @@ class ResponseEncoding:class Response(Adaptor):"""This class is returned by all engines as a way to unify response type between different libraries."""- def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict, encoding: str = 'utf-8', **adaptor_arguments: Dict):+ def __init__(self, url: str, text: str, body: bytes, status: int, reason: str, cookies: Dict, headers: Dict, request_headers: Dict,+ encoding: str = 'utf-8', method: str = 'GET', **adaptor_arguments: Dict):automatch_domain = adaptor_arguments.pop('automatch_domain', None)self.status = statusself.reason = reason@@ -96,6 +97,8 @@ class Response(Adaptor):super().__init__(text=text, body=body, url=automatch_domain or url, encoding=encoding, **adaptor_arguments)# For back-ward compatibilityself.adaptor = self+ # For easier debugging while working from a Python shell+ logging.info(f'Fetched ({status}) <{method} {url}> (referer: {request_headers.get("referer")})')# def __repr__(self):# return f'<{self.__class__.__name__} [{self.status} {self.reason}]>'