🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
perf(parser): make Selector.get_all_text() O(nodes) by precomputing ignored subtrees
yetvalf4fc6a1baa6d
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 8401f27..65924cc 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -300,7 +300,8 @@ class Selector(SelectorsGeneration):ignored_elements: set[Any] = set()if ignore_tags:- ignored_elements.update(self._root.iter(*ignore_tags))+ for element in self._root.iter(*ignore_tags):+ ignored_elements.update(element.iter())_all_strings = []@@ -315,11 +316,7 @@ class Selector(SelectorsGeneration):return Falseowner = parent.getparent() if text_node.is_tail else parent- while owner is not None:- if owner in ignored_elements:- return False- owner = owner.getparent()- return True+ return owner not in ignored_elementsfor text_node in cast(list[_ElementUnicodeResult], _find_all_text_nodes(self._root)):text = str(text_node)diff --git a/tests/parser/test_parser_advanced.py b/tests/parser/test_parser_advanced.pyindex 39829bb..050d89c 100644--- a/tests/parser/test_parser_advanced.py+++ b/tests/parser/test_parser_advanced.py@@ -210,6 +210,23 @@ class TestAdvancedSelectors:assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5\nstring6\nstring7"+ def test_get_all_text_keeps_text_after_comment(self):+ """Tail text following a comment or PI node must still be collected"""+ html = "<div>before<!-- c -->after<p>x</p></div>"+ assert Selector(html, keep_comments=True).get_all_text() == "before\nafter\nx"++ def test_get_all_text_deep_nesting(self):+ """Deeply nested trees collect every visible text node and skip ignored subtrees"""+ depth = 500+ html = "<div>" + "".join(f"<span>t{i}" for i in range(depth)) + "leaf" + "</span>" * depth + "end</div>"+ text = Selector(html).get_all_text(strip=True)+ assert text.startswith("t0\nt1\n")+ assert "leaf" in text and text.endswith("end")+ assert text.count("\n") == depth++ ignored = "<div>keep<script>" + "<b>x</b>" * depth + "</script>done</div>"+ assert Selector(ignored).get_all_text(strip=True) == "keep\ndone"+class TestTextHandlerAdvanced:"""Test advanced TextHandler functionality"""