🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(response): fix markdown concatenation (#432)
Yiğit ERDOĞAN2947fca6d4a7
diff --git a/scrapling/engines/toolbelt/custom.py b/scrapling/engines/toolbelt/custom.pyindex ea86233..ff2f244 100644--- a/scrapling/engines/toolbelt/custom.py+++ b/scrapling/engines/toolbelt/custom.py@@ -99,7 +99,7 @@ class Response(Selector):page = (cast(Selector, self.css("body").first) or self) if main_content_only else selfpage = Convertor._sanitize_for_ai(Convertor._strip_noise_tags(page))pages = [page] if not css_selector else page.css(css_selector)- return "".join(Convertor._convert_to_markdown(element.html_content) for element in pages)+ return "\n\n".join(Convertor._convert_to_markdown(element.html_content) for element in pages)def follow(self,diff --git a/tests/fetchers/test_response_markdown.py b/tests/fetchers/test_response_markdown.pyindex c725183..83f6dff 100644--- a/tests/fetchers/test_response_markdown.py+++ b/tests/fetchers/test_response_markdown.py@@ -21,10 +21,20 @@ HTML = """"""-def _make_response(url: str = "https://example.com/") -> Response:+SECTIONS_HTML = """+<html>+ <body>+ <article><h2>First section</h2><p>Body one.</p></article>+ <article><h2>Second section</h2><p>Body two.</p></article>+ </body>+</html>+"""+++def _make_response(url: str = "https://example.com/", content: str = HTML) -> Response:return Response(url=url,- content=HTML,+ content=content,status=200,reason="OK",cookies={},@@ -55,10 +65,15 @@ class TestResponseMarkdown:def test_css_selector_concatenates_all_matches(self):md = _make_response().markdown(css_selector=".a")- assert "First" in md- assert "Second" in md+ assert "First\n\nSecond" in mdassert "Visible content." not in md+ def test_css_selector_matches_do_not_bleed_into_each_other(self):+ """Each match is a separate Markdown block, so the last line of one can't join the next"""+ md = _make_response(content=SECTIONS_HTML).markdown(css_selector="article")+ assert "Body one.\n\nSecond section" in md+ assert "Body one.Second section" not in md+def test_missing_markdownify_raises_friendly_error(self, monkeypatch):monkeypatch.setitem(sys.modules, "markdownify", None)with pytest.raises(ModuleNotFoundError, match=r"scrapling\[rag\]"):