🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(parser): Multiple optimizations and fixes
Karim shoair297e14230bf1
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex ad83f26..3ff3f2c 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -59,6 +59,7 @@ class Adaptor(SelectorsGeneration):keep_comments: Optional[bool] = False,keep_cdata: Optional[bool] = False,auto_match: Optional[bool] = False,+ _storage: object = None,storage: Any = SQLiteStorageSystem,storage_args: Optional[Dict] = None,**kwargs,@@ -136,25 +137,28 @@ class Adaptor(SelectorsGeneration):self.__auto_match_enabled = auto_matchif self.__auto_match_enabled:- if not storage_args:- storage_args = {- "storage_file": os.path.join(- os.path.dirname(__file__), "elements_storage.db"- ),- "url": url,- }-- if not hasattr(storage, "__wrapped__"):- raise ValueError(- "Storage class must be wrapped with lru_cache decorator, see docs for info"- )+ if _storage is not None:+ self._storage = _storage+ else:+ if not storage_args:+ storage_args = {+ "storage_file": os.path.join(+ os.path.dirname(__file__), "elements_storage.db"+ ),+ "url": url,+ }++ if not hasattr(storage, "__wrapped__"):+ raise ValueError(+ "Storage class must be wrapped with lru_cache decorator, see docs for info"+ )- if not issubclass(storage.__wrapped__, StorageSystemMixin):- raise ValueError(- "Storage system must be inherited from class `StorageSystemMixin`"- )+ if not issubclass(storage.__wrapped__, StorageSystemMixin):+ raise ValueError(+ "Storage system must be inherited from class `StorageSystemMixin`"+ )- self._storage = storage(**storage_args)+ self._storage = storage(**storage_args)self.__keep_comments = keep_commentsself.__keep_cdata = keep_cdata@@ -186,6 +190,8 @@ class Adaptor(SelectorsGeneration):if hasattr(self, "_storage") and self._storage:try:self._storage.close()+ except Exception:+ passfinally:self._storage = None@@ -214,13 +220,15 @@ class Adaptor(SelectorsGeneration):def __element_convertor(self, element: html.HtmlElement) -> "Adaptor":"""Used internally to convert a single HtmlElement to Adaptor directly without checks"""+ db_instance = (+ self._storage if (hasattr(self, "_storage") and self._storage) else None+ )return Adaptor(root=element,- text="",- body=b"", # Since the root argument is provided, both `text` and `body` will be ignored, so this is just a fillerurl=self.url,encoding=self.encoding,auto_match=self.__auto_match_enabled,+ _storage=db_instance, # Reuse existing storage if it exists otherwise it won't be checked if `auto_match` is turned offkeep_comments=self.__keep_comments,keep_cdata=self.__keep_cdata,huge_tree=self.__huge_tree_enabled,@@ -630,8 +638,10 @@ class Adaptor(SelectorsGeneration):except (SelectorError,SelectorSyntaxError,- ):- raise SelectorSyntaxError(f"Invalid CSS selector: {selector}")+ ) as e:+ raise SelectorSyntaxError(+ f"Invalid CSS selector '{selector}': {str(e)}"+ ) from edef xpath(self,@@ -700,8 +710,8 @@ class Adaptor(SelectorsGeneration):SelectorSyntaxError,etree.XPathError,etree.XPathEvalError,- ):- raise SelectorSyntaxError(f"Invalid XPath selector: {selector}")+ ) as e:+ raise SelectorSyntaxError(f"Invalid XPath selector: {selector}") from edef find_all(self,