🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(cli): write md and txt extract output as UTF-8 (#488)
Suliman Abdulrazzaq71f567e8068e
diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 1c85f72..779b02a 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -701,12 +701,15 @@ class Convertor:elif not filename.endswith((".md", ".html", ".txt")):raise ValueError("Unknown file type: filename must end with '.md', '.html', or '.txt'")else:- with open(filename, "w", encoding=page.encoding) as f:- extension = filename.split(".")[-1]- content = cls._extract_content(- page,- cls._extension_map[extension],- css_selector=css_selector,- main_content_only=main_content_only,- )- f.write(("\n\n" if extension == "md" else "").join(filter(None, content)))+ extension = filename.split(".")[-1]+ content = cls._extract_content(+ page,+ cls._extension_map[extension],+ css_selector=css_selector,+ main_content_only=main_content_only,+ )+ # Build the whole text before opening the file so a failure can't leave an empty file behind+ text = ("\n\n" if extension == "md" else "").join(filter(None, content))+ # Markdown and text are produced by the converter, so they don't depend on the page charset+ with open(filename, "w", encoding=page.encoding if extension == "html" else "utf-8") as f:+ f.write(text)diff --git a/tests/cli/test_shell_functionality.py b/tests/cli/test_shell_functionality.pyindex 995e657..5b0de7d 100644--- a/tests/cli/test_shell_functionality.py+++ b/tests/cli/test_shell_functionality.py@@ -153,6 +153,52 @@ class TestConvertor:Convertor.write_content_to_file(page, str(txt_file))assert txt_file.exists()+ @pytest.mark.parametrize("encoding", ["iso-8859-1", "windows-1252"])+ @pytest.mark.parametrize("extension", ["md", "txt"])+ def test_write_text_outputs_as_utf8(self, encoding, extension, tmp_path):+ """Test that md/txt output does not depend on the page charset"""+ page = Selector("<html><body><p>Café — 中</p></body></html>", encoding=encoding)++ out_file = tmp_path / f"output.{extension}"+ Convertor.write_content_to_file(page, str(out_file))++ assert out_file.read_bytes().decode("utf-8").strip() == "CafĂ© — ä¸"++ def test_write_html_keeps_page_encoding(self, tmp_path):+ """Test that html output is still written with the page encoding"""+ page = Selector("<html><body><p>Café</p></body></html>", encoding="iso-8859-1")++ out_file = tmp_path / "output.html"+ Convertor.write_content_to_file(page, str(out_file))++ assert "Caf\xe9".encode("iso-8859-1") in out_file.read_bytes()++ def test_write_utf8_page_is_unchanged(self, tmp_path):+ """Test that md/txt output of a UTF-8 page is written as UTF-8"""+ page = Selector("<html><body><p>CafĂ© — ä¸</p></body></html>")++ for extension in ("md", "txt"):+ out_file = tmp_path / f"output.{extension}"+ Convertor.write_content_to_file(page, str(out_file))+ assert out_file.read_bytes().decode("utf-8").strip() == "CafĂ© — ä¸"++ def test_failed_conversion_keeps_existing_file(self, sample_html, tmp_path):+ """Test that a failing conversion does not truncate the output file"""+ page = Selector(sample_html)+ out_file = tmp_path / "output.md"+ out_file.write_text("previous content", encoding="utf-8")++ with patch.object(Convertor, "_extract_content", side_effect=RuntimeError("boom")):+ with pytest.raises(RuntimeError):+ Convertor.write_content_to_file(page, str(out_file))+ assert out_file.read_text(encoding="utf-8") == "previous content"++ missing_file = tmp_path / "missing.txt"+ with patch.object(Convertor, "_extract_content", side_effect=RuntimeError("boom")):+ with pytest.raises(RuntimeError):+ Convertor.write_content_to_file(page, str(missing_file))+ assert not missing_file.exists()+def test_invalid_operations(self, sample_html):"""Test error handling in convertor"""page = Selector(sample_html)