Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(cli): fix markdown concatenation

Karim shoair20337ba6a518
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index c9aa2cf..5f189a0 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -673,13 +673,10 @@ class Convertor:
else:
with open(filename, "w", encoding=page.encoding) as f:
extension = filename.split(".")[-1]
- f.write(
- "".join(
- cls._extract_content(
- page,
- cls._extension_map[extension],
- css_selector=css_selector,
- main_content_only=main_content_only,
- )
- )
+ content = cls._extract_content(
+ page,
+ cls._extension_map[extension],
+ css_selector=css_selector,
+ main_content_only=main_content_only,
)
+ f.write(("\n\n" if extension == "md" else "").join(filter(None, content)))
diff --git a/tests/cli/test_markdown_blocks.py b/tests/cli/test_markdown_blocks.py
new file mode 100644
index 0000000..6a7fcdf
--- /dev/null
+++ b/tests/cli/test_markdown_blocks.py
@@ -0,0 +1,21 @@
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from scrapling.cli import main
+from scrapling.parser import Selector
+
+
+ ("extension", "expected"),
+ [("md", "First\n\nSecond"), ("txt", "FirstSecond"), ("html", "<p>First</p><p></p><p>Second</p>")],
+)
+def test_cli_preserves_selected_blocks(tmp_path: Path, extension: str, expected: str) -> None:
+ page = Selector("<html><body><p>First</p><p></p><p>Second</p></body></html>")
+ output = tmp_path / f"output.{extension}"
+ with patch("scrapling.fetchers.Fetcher.get", return_value=page):
+ result = CliRunner().invoke(main, ["extract", "get", "https://example.com", str(output), "-s", "p"])
+ assert result.exit_code == 0, result.output
+ assert output.read_text(encoding="utf-8") == expected