🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: Selector.get_all_text() doesn't get all text #167
Matt Hillebrand5457a697faec
diff --git a/scrapling/parser.py b/scrapling/parser.pyindex 166e6ea..e510fdf 100644--- a/scrapling/parser.py+++ b/scrapling/parser.py@@ -304,13 +304,23 @@ class Selector(SelectorsGeneration):ignored_elements.update(cast(list, _find_all_elements(element)))_all_strings = []- for node in self._root.iter():- if node not in ignored_elements:- text = node.text- if text and isinstance(text, str):- processed_text = text.strip() if strip else text- if not valid_values or processed_text.strip():- _all_strings.append(processed_text)++ def append_text(text: Any) -> None:+ if text and isinstance(text, str):+ processed_text = text.strip() if strip else text+ if not valid_values or processed_text.strip():+ _all_strings.append(processed_text)++ def walk(node: Any) -> None:+ if node in ignored_elements:+ return++ append_text(node.text)+ for child in node:+ walk(child)+ append_text(child.tail)++ walk(self._root)return cast(TextHandler, TextHandler(separator).join(_all_strings))diff --git a/tests/parser/test_general.py b/tests/parser/test_general.pyindex 26f6e6c..d459d44 100644--- a/tests/parser/test_general.py+++ b/tests/parser/test_general.py@@ -327,6 +327,30 @@ def test_getting_all_text(page):assert page.get_all_text() != ""+def test_getting_all_text_from_nested_content():+ """Test getting all text preserves interleaved text nodes"""+ html = """+ <html>+ <body>+ <main>+ string1+ <b>string2</b>+ string3+ <div>+ <span>string4</span>+ </div>+ string5+ </main>+ </body>+ </html>+ """++ page = Selector(html, adaptive=False)+ node = page.css("main")[0]++ assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5"++def test_regex_on_text(page):"""Test regex operations on text"""element = page.css('[data-id="1"] .price')[0]