🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style: Adjustments to the translator
Karim shoair32cb76604c1e
diff --git a/scrapling/core/translator.py b/scrapling/core/translator.pyindex 9c987ff..bdab7a5 100644--- a/scrapling/core/translator.py+++ b/scrapling/core/translator.py@@ -8,7 +8,7 @@ So you don't have to learn a new selectors/api method like what bs4 done with soIf you want to learn about this, head to https://cssselect.readthedocs.io/en/latest/#cssselect.FunctionalPseudoElement"""-import re+from functools import lru_cachefrom cssselect import HTMLTranslator as OriginalHTMLTranslatorfrom cssselect.parser import Element, FunctionalPseudoElement, PseudoElement@@ -16,11 +16,6 @@ from cssselect.xpath import ExpressionErrorfrom cssselect.xpath import XPathExpr as OriginalXPathExprfrom scrapling.core._types import Any, Optional, Protocol, Self-from scrapling.core.utils import lru_cache--HTML5_WHITESPACE = " \t\n\r\x0c" # From w3lib.html.HTML5_WHITESPACE-regex = f"[{HTML5_WHITESPACE}]+"-replace_html5_whitespaces = re.compile(regex).subclass XPathExpr(OriginalXPathExpr):@@ -33,7 +28,7 @@ class XPathExpr(OriginalXPathExpr):xpath: OriginalXPathExpr,textnode: bool = False,attribute: Optional[str] = None,- ) -> "Self":+ ) -> Self:x = cls(path=xpath.path, element=xpath.element, condition=xpath.condition)x.textnode = textnodex.attribute = attribute@@ -57,12 +52,12 @@ class XPathExpr(OriginalXPathExpr):return pathdef join(- self: "Self",+ self: Self,combiner: str,other: OriginalXPathExpr,*args: Any,**kwargs: Any,- ) -> "Self":+ ) -> Self:if not isinstance(other, XPathExpr):raise ValueError(f"Expressions of type {__name__}.XPathExpr can ony join expressions"@@ -90,7 +85,7 @@ class TranslatorMixin:"""def xpath_element(self: TranslatorProtocol, selector: Element) -> XPathExpr:- # https://github.com/python/mypy/issues/12344+ # https://github.com/python/mypy/issues/14757xpath = super().xpath_element(selector) # type: ignore[safe-super]return XPathExpr.from_xpath(xpath)@@ -98,7 +93,7 @@ class TranslatorMixin:self, xpath: OriginalXPathExpr, pseudo_element: PseudoElement) -> OriginalXPathExpr:"""- Dispatch method that transforms XPath to support pseudo-elements.+ Dispatch method that transforms XPath to support the pseudo-element."""if isinstance(pseudo_element, FunctionalPseudoElement):method_name = f"xpath_{pseudo_element.name.replace('-', '_')}_functional_pseudo_element"@@ -143,4 +138,4 @@ class HTMLTranslator(TranslatorMixin, OriginalHTMLTranslator):return super().css_to_xpath(css, prefix)-translator_instance = HTMLTranslator()+translator = HTMLTranslator()diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 219181d..7540ba7 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -36,7 +36,7 @@ from scrapling.core.storage import (StorageSystemMixin,_StorageTools,)-from scrapling.core.translator import translator_instance+from scrapling.core.translator import translator as _translatorfrom scrapling.core.utils import clean_spaces, flatten, html_forbidden, is_jsonable, log__DEFAULT_DB_FILE__ = str(Path(__file__).parent / "elements_storage.db")@@ -600,7 +600,7 @@ class Selector(SelectorsGeneration):try:if not self.__adaptive_enabled or "," not in selector:# No need to split selectors in this case, let's save some CPU cycles :)- xpath_selector = translator_instance.css_to_xpath(selector)+ xpath_selector = _translator.css_to_xpath(selector)return self.xpath(xpath_selector,identifier or selector,@@ -614,7 +614,7 @@ class Selector(SelectorsGeneration):for single_selector in split_selectors(selector):# I'm doing this only so the `save` function saves data correctly for combined selectors# Like using the ',' to combine two different selectors that point to different elements.- xpath_selector = translator_instance.css_to_xpath(+ xpath_selector = _translator.css_to_xpath(single_selector.canonical())results += self.xpath(