Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

perf(parser): make Selector.get_all_text() O(nodes) by precomputing ignored subtrees

yetvalf4fc6a1baa6d
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 8401f27..65924cc 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -300,7 +300,8 @@ class Selector(SelectorsGeneration):
ignored_elements: set[Any] = set()
if ignore_tags:
- ignored_elements.update(self._root.iter(*ignore_tags))
+ for element in self._root.iter(*ignore_tags):
+ ignored_elements.update(element.iter())
_all_strings = []
@@ -315,11 +316,7 @@ class Selector(SelectorsGeneration):
return False
owner = parent.getparent() if text_node.is_tail else parent
- while owner is not None:
- if owner in ignored_elements:
- return False
- owner = owner.getparent()
- return True
+ return owner not in ignored_elements
for text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)):
text = str(text_node)
diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.py
index 39829bb..050d89c 100644
--- a/tests/parser/test_parser_advanced.py
+++ b/tests/parser/test_parser_advanced.py
@@ -210,6 +210,23 @@ class TestAdvancedSelectors:
assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5\nstring6\nstring7"
+ def test_get_all_text_keeps_text_after_comment(self):
+ """Tail text following a comment or PI node must still be collected"""
+ html = "<div>before<!-- c -->after<p>x</p></div>"
+ assert Selector(html, keep_comments=True).get_all_text() == "before\nafter\nx"
+
+ def test_get_all_text_deep_nesting(self):
+ """Deeply nested trees collect every visible text node and skip ignored subtrees"""
+ depth = 500
+ html = "<div>" + "".join(f"<span>t{i}" for i in range(depth)) + "leaf" + "</span>" * depth + "end</div>"
+ text = Selector(html).get_all_text(strip=True)
+ assert text.startswith("t0\nt1\n")
+ assert "leaf" in text and text.endswith("end")
+ assert text.count("\n") == depth
+
+ ignored = "<div>keep<script>" + "<b>x</b>" * depth + "</script>done</div>"
+ assert Selector(ignored).get_all_text(strip=True) == "keep\ndone"
+
class TestTextHandlerAdvanced:
"""Test advanced TextHandler functionality"""