Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

perf: Speed up `clean` functions

Karim shoair3c5de8e0f2b8
diff --git a/scrapling/core/custom_types.py b/scrapling/core/custom_types.py
index 2314556..79d6e6c 100644
--- a/scrapling/core/custom_types.py
+++ b/scrapling/core/custom_types.py
@@ -18,11 +18,12 @@ from scrapling.core._types import (
Generator,
SupportsIndex,
)
-from scrapling.core.utils import _is_iterable, flatten
+from scrapling.core.utils import _is_iterable, flatten, __CONSECUTIVE_SPACES_REGEX__
from scrapling.core._html_utils import _replace_entities
# Define type variable for AttributeHandler value type
_TextHandlerType = TypeVar("_TextHandlerType", bound="TextHandler")
+__CLEANING_TABLE__ = str.maketrans("\t\r\n", " ")
class TextHandler(str):
@@ -118,9 +119,8 @@ class TextHandler(str):
def clean(self) -> Union[str, "TextHandler"]:
"""Return a new version of the string after removing all white spaces and consecutive spaces"""
- trans_table = str.maketrans("\t\r\n", " ")
- data = self.translate(trans_table)
- return self.__class__(sub(" +", " ", data).strip())
+ data = self.translate(__CLEANING_TABLE__)
+ return self.__class__(__CONSECUTIVE_SPACES_REGEX__.sub(" ", data).strip())
# For easy copy-paste from Scrapy/parsel code when needed :)
def get(self, default=None):
diff --git a/scrapling/core/utils.py b/scrapling/core/utils.py
index 0219cb0..fce40d8 100644
--- a/scrapling/core/utils.py
+++ b/scrapling/core/utils.py
@@ -14,6 +14,9 @@ html_forbidden = {
html.HtmlComment,
}
+__CLEANING_TABLE__ = str.maketrans({"\t": " ", "\n": None, "\r": None})
+__CONSECUTIVE_SPACES_REGEX__ = re.compile(r" +")
+
@lru_cache(1, typed=True)
def setup_logger():
@@ -135,6 +138,5 @@ class _StorageTools:
@lru_cache(128, typed=True)
def clean_spaces(string):
- string = string.replace("\t", " ")
- string = re.sub("[\n|\r]", "", string)
- return re.sub(" +", " ", string)
+ string = string.translate(__CLEANING_TABLE__)
+ return __CONSECUTIVE_SPACES_REGEX__.sub(" ", string)