🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(parser): Improving CSS selecting memory usage by ~144%
Karim shoaire5a022ec6acb
diff --git a/scrapling/core/translator.py b/scrapling/core/translator.pyindex 263a24a..d2d03fa 100644--- a/scrapling/core/translator.py+++ b/scrapling/core/translator.py@@ -142,3 +142,6 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):@lru_cache(maxsize=256)def css_to_xpath(self, css: str, prefix: str = "descendant-or-self::") -> str:return super().css_to_xpath(css, prefix)+++translator_instance = HTMLTranslator()diff --git a/scrapling/parser.py b/scrapling/parser.pyindex f9e177e..146ea47 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -17,7 +17,7 @@ from scrapling.core.custom_types import (AttributesHandler, TextHandler,from scrapling.core.mixins import SelectorsGenerationfrom scrapling.core.storage_adaptors import (SQLiteStorageSystem,StorageSystemMixin, _StorageTools)-from scrapling.core.translator import HTMLTranslator+from scrapling.core.translator import translator_instancefrom scrapling.core.utils import (clean_spaces, flatten, html_forbidden,is_jsonable, log)@@ -476,7 +476,7 @@ class Adaptor(SelectorsGeneration):try:if not self.__auto_match_enabled or ',' not in selector:# No need to split selectors in this case, let's save some CPU cycles :)- xpath_selector = HTMLTranslator().css_to_xpath(selector)+ xpath_selector = translator_instance.css_to_xpath(selector)return self.xpath(xpath_selector, identifier or selector, auto_match, auto_save, percentage)results = []@@ -484,7 +484,7 @@ class Adaptor(SelectorsGeneration):for single_selector in split_selectors(selector):# I'm doing this only so the `save` function save data correctly for combined selectors# Like using the ',' to combine two different selectors that point to different elements.- xpath_selector = HTMLTranslator().css_to_xpath(single_selector.canonical())+ xpath_selector = translator_instance.css_to_xpath(single_selector.canonical())results += self.xpath(xpath_selector, identifier or single_selector.canonical(), auto_match, auto_save, percentage)