🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(cli): fix markdown concatenation
Karim shoair20337ba6a518
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex c9aa2cf..5f189a0 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -673,13 +673,10 @@ class Convertor:else:with open(filename, "w", encoding=page.encoding) as f:extension = filename.split(".")[-1]- f.write(- "".join(- cls._extract_content(- page,- cls._extension_map[extension],- css_selector=css_selector,- main_content_only=main_content_only,- )- )+ content = cls._extract_content(+ page,+ cls._extension_map[extension],+ css_selector=css_selector,+ main_content_only=main_content_only,)+ f.write(("\n\n" if extension == "md" else "").join(filter(None, content)))diff --git a/tests/cli/test_markdown_blocks.py b/tests/cli/test_markdown_blocks.pynew file mode 100644index 0000000..6a7fcdf--- /dev/null+++ b/tests/cli/test_markdown_blocks.py@@ -0,0 +1,21 @@+from pathlib import Path+from unittest.mock import patch++import pytest+from click.testing import CliRunner++from scrapling.cli import main+from scrapling.parser import Selector+++ ("extension", "expected"),+ [("md", "First\n\nSecond"), ("txt", "FirstSecond"), ("html", "<p>First</p><p></p><p>Second</p>")],+)+def test_cli_preserves_selected_blocks(tmp_path: Path, extension: str, expected: str) -> None:+ page = Selector("<html><body><p>First</p><p></p><p>Second</p></body></html>")+ output = tmp_path / f"output.{extension}"+ with patch("scrapling.fetchers.Fetcher.get", return_value=page):+ result = CliRunner().invoke(main, ["extract", "get", "https://example.com", str(output), "-s", "p"])+ assert result.exit_code == 0, result.output+ assert output.read_text(encoding="utf-8") == expected