Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(response): fix markdown concatenation (#432)

Yiğit ERDOĞAN2947fca6d4a7
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.py
index ea86233..ff2f244 100644
--- a/scrapling/engines/toolbelt/custom.py
+++ b/scrapling/engines/toolbelt/custom.py
@@ -99,7 +99,7 @@ class Response(Selector):
page = (cast(Selector, self.css("body").first) or self) if main_content_only else self
page = Convertor._sanitize_for_ai(Convertor._strip_noise_tags(page))
pages = [page] if not css_selector else page.css(css_selector)
- return "".join(Convertor._convert_to_markdown(element.html_content) for element in pages)
+ return "\n\n".join(Convertor._convert_to_markdown(element.html_content) for element in pages)
def follow(
self,
diff --git a/tests/fetchers/test_response_markdown.py b/tests/fetchers/test_response_markdown.py
index c725183..83f6dff 100644
--- a/tests/fetchers/test_response_markdown.py
+++ b/tests/fetchers/test_response_markdown.py
@@ -21,10 +21,20 @@ HTML = """
"""
-def _make_response(url: str = "https://example.com/") -> Response:
+SECTIONS_HTML = """
+<html>
+ <body>
+ <article><h2>First section</h2><p>Body one.</p></article>
+ <article><h2>Second section</h2><p>Body two.</p></article>
+ </body>
+</html>
+"""
+
+
+def _make_response(url: str = "https://example.com/", content: str = HTML) -> Response:
return Response(
url=url,
- content=HTML,
+ content=content,
status=200,
reason="OK",
cookies={},
@@ -55,10 +65,15 @@ class TestResponseMarkdown:
def test_css_selector_concatenates_all_matches(self):
md = _make_response().markdown(css_selector=".a")
- assert "First" in md
- assert "Second" in md
+ assert "First\n\nSecond" in md
assert "Visible content." not in md
+ def test_css_selector_matches_do_not_bleed_into_each_other(self):
+ """Each match is a separate Markdown block, so the last line of one can't join the next"""
+ md = _make_response(content=SECTIONS_HTML).markdown(css_selector="article")
+ assert "Body one.\n\nSecond section" in md
+ assert "Body one.Second section" not in md
+
def test_missing_markdownify_raises_friendly_error(self, monkeypatch):
monkeypatch.setitem(sys.modules, "markdownify", None)
with pytest.raises(ModuleNotFoundError, match=r"scrapling\[rag\]"):