Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

perf: optimize `get_all_text` and adaptive logic by another 10%

Karim shoaird1a2ecd34126
diff --git a/scrapling/parser.py b/scrapling/parser.py
index e3d30c6..84e7876 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -323,7 +323,7 @@ class Selector(SelectorsGeneration):
ignored_elements.update(set(element.iterchildren()))
_all_strings = []
- for node in self._root.xpath(".//*"):
+ for node in self._root.iter():
if node not in ignored_elements:
text = node.text
if text and isinstance(text, str):
@@ -496,7 +496,7 @@ class Selector(SelectorsGeneration):
if issubclass(type(element), HtmlElement):
element = _StorageTools.element_to_dict(element)
- for node in self._root.xpath(".//*"):
+ for node in self._root.iter("*"):
# Collect all elements in the page, then for each element get the matching score of it against the node.
# Hence: the code doesn't stop even if the score was 100%
# because there might be another element(s) left in page with the same score