🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
perf: Speed up `clean` functions
Karim shoair3c5de8e0f2b8
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.pyindex 2314556..79d6e6c 100644--- a/scrapling/core/custom_types.py+++ b/scrapling/core/custom_types.py@@ -18,11 +18,12 @@ from scrapling.core._types import (Generator,SupportsIndex,)-from scrapling.core.utils import _is_iterable, flatten+from scrapling.core.utils import _is_iterable, flatten, __CONSECUTIVE_SPACES_REGEX__from scrapling.core._html_utils import _replace_entities# Define type variable for AttributeHandler value type_TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")+__CLEANING_TABLE__ = str.maketrans("\t\r\n", " ")class TextHandler(str):@@ -118,9 +119,8 @@ class TextHandler(str):def clean(self) -> Union[str, "TextHandler"]:"""Return a new version of the string after removing all white spaces and consecutive spaces"""- trans_table = str.maketrans("\t\r\n", " ")- data = self.translate(trans_table)- return self.__class__(sub(" +", " ", data).strip())+ data = self.translate(__CLEANING_TABLE__)+ return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())# For easy copy-paste from Scrapy/parsel code when needed :)def get(self, default=None):diff --git a/scrapling/core/utils.py b/scrapling/core/utils.pyindex 0219cb0..fce40d8 100644--- a/scrapling/core/utils.py+++ b/scrapling/core/utils.py@@ -14,6 +14,9 @@ html_forbidden = {html.HtmlComment,}+__CLEANING_TABLE__ = str.maketrans({"\t": " ", "\n": None, "\r": None})+__CONSECUTIVE_SPACES_REGEX__ = re.compile(r" +")+@lru_cache(1, typed=True)def setup_logger():@@ -135,6 +138,5 @@ class _StorageTools:@lru_cache(128, typed=True)def clean_spaces(string):- string = string.replace("\t", " ")- string = re.sub("[\n|\r]", "", string)- return re.sub(" +", " ", string)+ string = string.translate(__CLEANING_TABLE__)+ return __CONSECUTIVE_SPACES_REGEX__.sub(" ", string)