Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(cli): write md and txt extract output as UTF-8 (#488)

Suliman Abdulrazzaq71f567e8068e
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.py
index 1c85f72..779b02a 100644
--- a/scrapling/core/shell.py
+++ b/scrapling/core/shell.py
@@ -701,12 +701,15 @@ class Convertor:
elif not filename.endswith((".md", ".html", ".txt")):
raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'")
else:
- with open(filename, "w", encoding=page.encoding) as f:
- extension = filename.split(".")[-1]
- content = cls._extract_content(
- page,
- cls._extension_map[extension],
- css_selector=css_selector,
- main_content_only=main_content_only,
- )
- f.write(("\n\n" if extension == "md" else "").join(filter(None, content)))
+ extension = filename.split(".")[-1]
+ content = cls._extract_content(
+ page,
+ cls._extension_map[extension],
+ css_selector=css_selector,
+ main_content_only=main_content_only,
+ )
+ # Build the whole text before opening the file so a failure can't leave an empty file behind
+ text = ("\n\n" if extension == "md" else "").join(filter(None, content))
+ # Markdown and text are produced by the converter, so they don't depend on the page charset
+ with open(filename, "w", encoding=page.encoding if extension == "html" else "utf-8") as f:
+ f.write(text)
diff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.py
index 995e657..5b0de7d 100644
--- a/tests/cli/test_shell_functionality.py
+++ b/tests/cli/test_shell_functionality.py
@@ -153,6 +153,52 @@ class TestConvertor:
Convertor.write_content_to_file(page, str(txt_file))
assert txt_file.exists()
+ @pytest.mark.parametrize("encoding", ["iso-8859-1", "windows-1252"])
+ @pytest.mark.parametrize("extension", ["md", "txt"])
+ def test_write_text_outputs_as_utf8(self, encoding, extension, tmp_path):
+ """Test that md/txt output does not depend on the page charset"""
+ page = Selector("<html><body><p>Caf&eacute; &mdash; &#x4E2D;</p></body></html>", encoding=encoding)
+
+ out_file = tmp_path / f"output.{extension}"
+ Convertor.write_content_to_file(page, str(out_file))
+
+ assert out_file.read_bytes().decode("utf-8").strip() == "Café — 中"
+
+ def test_write_html_keeps_page_encoding(self, tmp_path):
+ """Test that html output is still written with the page encoding"""
+ page = Selector("<html><body><p>Caf&eacute;</p></body></html>", encoding="iso-8859-1")
+
+ out_file = tmp_path / "output.html"
+ Convertor.write_content_to_file(page, str(out_file))
+
+ assert "Caf\xe9".encode("iso-8859-1") in out_file.read_bytes()
+
+ def test_write_utf8_page_is_unchanged(self, tmp_path):
+ """Test that md/txt output of a UTF-8 page is written as UTF-8"""
+ page = Selector("<html><body><p>Café — 中</p></body></html>")
+
+ for extension in ("md", "txt"):
+ out_file = tmp_path / f"output.{extension}"
+ Convertor.write_content_to_file(page, str(out_file))
+ assert out_file.read_bytes().decode("utf-8").strip() == "Café — 中"
+
+ def test_failed_conversion_keeps_existing_file(self, sample_html, tmp_path):
+ """Test that a failing conversion does not truncate the output file"""
+ page = Selector(sample_html)
+ out_file = tmp_path / "output.md"
+ out_file.write_text("previous content", encoding="utf-8")
+
+ with patch.object(Convertor, "_extract_content", side_effect=RuntimeError("boom")):
+ with pytest.raises(RuntimeError):
+ Convertor.write_content_to_file(page, str(out_file))
+ assert out_file.read_text(encoding="utf-8") == "previous content"
+
+ missing_file = tmp_path / "missing.txt"
+ with patch.object(Convertor, "_extract_content", side_effect=RuntimeError("boom")):
+ with pytest.raises(RuntimeError):
+ Convertor.write_content_to_file(page, str(missing_file))
+ assert not missing_file.exists()
+
def test_invalid_operations(self, sample_html):
"""Test error handling in convertor"""
page = Selector(sample_html)