Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: Selector.get_all_text() doesn't get all text #167

Matt Hillebrand5457a697faec
diff --git a/scrapling/parser.py b/scrapling/parser.py
index 166e6ea..e510fdf 100644
--- a/scrapling/parser.py
+++ b/scrapling/parser.py
@@ -304,13 +304,23 @@ class Selector(SelectorsGeneration):
ignored_elements.update(cast(list, _find_all_elements(element)))
_all_strings = []
- for node in self._root.iter():
- if node not in ignored_elements:
- text = node.text
- if text and isinstance(text, str):
- processed_text = text.strip() if strip else text
- if not valid_values or processed_text.strip():
- _all_strings.append(processed_text)
+
+ def append_text(text: Any) -> None:
+ if text and isinstance(text, str):
+ processed_text = text.strip() if strip else text
+ if not valid_values or processed_text.strip():
+ _all_strings.append(processed_text)
+
+ def walk(node: Any) -> None:
+ if node in ignored_elements:
+ return
+
+ append_text(node.text)
+ for child in node:
+ walk(child)
+ append_text(child.tail)
+
+ walk(self._root)
return cast(TextHandler, TextHandler(separator).join(_all_strings))
diff --git a/tests/parser/test_general.py b/tests/parser/test_general.py
index 26f6e6c..d459d44 100644
--- a/tests/parser/test_general.py
+++ b/tests/parser/test_general.py
@@ -327,6 +327,30 @@ def test_getting_all_text(page):
assert page.get_all_text() != ""
+def test_getting_all_text_from_nested_content():
+ """Test getting all text preserves interleaved text nodes"""
+ html = """
+ <html>
+ <body>
+ <main>
+ string1
+ <b>string2</b>
+ string3
+ <div>
+ <span>string4</span>
+ </div>
+ string5
+ </main>
+ </body>
+ </html>
+ """
+
+ page = Selector(html, adaptive=False)
+ node = page.css("main")[0]
+
+ assert node.get_all_text("\n", strip=True) == "string1\nstring2\nstring3\nstring4\nstring5"
+
+
def test_regex_on_text(page):
"""Test regex operations on text"""
element = page.css('[data-id="1"] .price')[0]