🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Making structure clearer to help with the next version
Karim shoair327281c66431
diff --git a/scrapling/utils.py b/scrapling/utils/__init__.pysimilarity index 58%rename from scrapling/utils.pyrename to scrapling/utils/__init__.pyindex 7eea9fa..3defaf4 100644--- a/scrapling/utils.py+++ b/scrapling/utils/__init__.py@@ -1,8 +1,6 @@import re-import osimport loggingfrom itertools import chain-from logging import handlers# Using cache on top of a class is brilliant way to achieve Singleton design pattern without much codefrom functools import lru_cache as cache # functools.cache is available on Python 3.9+ only so let's keep lru_cache@@ -45,64 +43,6 @@ def _is_iterable(s: Any):return isinstance(s, (list, tuple,))-@cache(None, typed=True)-class _Logger(object):- # I will leave this class here for now in case I decide I want to come back to use it :)- __slots__ = ('console_logger', 'logger_file_path',)- levels = {- 'debug': logging.DEBUG,- 'info': logging.INFO,- 'warning': logging.WARNING,- 'error': logging.ERROR,- 'critical': logging.CRITICAL- }-- def __init__(self, filename: str = 'debug.log', level: str = 'debug', when: str = 'midnight', backcount: int = 1):- os.makedirs(os.path.join(os.path.dirname(__file__), 'logs'), exist_ok=True)- format_str = logging.Formatter("[%(asctime)s] %(levelname)s: %(message)s", "%Y-%m-%d %H:%M:%S")-- # on-screen output- lvl = self.levels[level.lower()]- self.console_logger = logging.getLogger('Scrapling')- self.console_logger.setLevel(lvl)- console_handler = logging.StreamHandler()- console_handler.setLevel(lvl)- console_handler.setFormatter(format_str)- self.console_logger.addHandler(console_handler)-- if lvl == logging.DEBUG:- filename = os.path.join(os.path.dirname(__file__), 'logs', filename)- self.logger_file_path = filename- # Automatically generates the logging file at specified intervals- file_handler = handlers.TimedRotatingFileHandler(- # If more than (backcount+1) existed, oldest logs will be deleted- filename=filename, when=when, backupCount=backcount, encoding='utf-8'- )- file_handler.setLevel(lvl)- file_handler.setFormatter(format_str)- # This for the logger when it appends the date to the new log- file_handler.namer = lambda name: name.replace(".log", "") + ".log"- self.console_logger.addHandler(file_handler)- self.debug(f'Debug log path: {self.logger_file_path}')- else:- self.logger_file_path = None-- def debug(self, message: str) -> None:- self.console_logger.debug(message)-- def info(self, message: str) -> None:- self.console_logger.info(message)-- def warning(self, message: str) -> None:- self.console_logger.warning(message)-- def error(self, message: str) -> None:- self.console_logger.error(message)-- def critical(self, message: str) -> None:- self.console_logger.critical(message)--class _StorageTools:@staticmethoddef __clean_attributes(element: html.HtmlElement, forbidden: tuple = ()) -> Dict: