🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(parser/get_all_text): Cleaner and a bit faster implementation
Karim shoair7c35341d85cf
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex eac0d64..5a4d4f5 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -223,29 +223,16 @@ class Adaptor(SelectorsGeneration)::return: A TextHandler"""_all_strings = []-- def _traverse(node: html.HtmlElement) -> None:- """Traverse element children and get text content of each-- :param node: Current node in the tree structure- :return:- """+ for node in self._root.xpath('.//*'):if node.tag not in ignore_tags:text = node.textif text and type(text) is str:- if valid_values:- if text.strip():- _all_strings.append(text if not strip else text.strip())+ if valid_values and text.strip():+ _all_strings.append(text if not strip else text.strip())else:_all_strings.append(text if not strip else text.strip())- for branch in node.iterchildren():- _traverse(branch)-- # We will start using Lxml directly for the speed boost- _traverse(self._root)-- return TextHandler(separator.join([s for s in _all_strings]))+ return TextHandler(separator.join(_all_strings))def urljoin(self, relative_url: str) -> str:"""Join this Adaptor's url with a relative url to form an absolute full URL."""