🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
Fix Playwright page content encoding
Yehuda_LevS0ae508f72583
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.pyindex cf303c8..e875cf4 100644--- a/scrapling/engines/toolbelt/convertor.py+++ b/scrapling/engines/toolbelt/convertor.py@@ -122,6 +122,7 @@ class ResponseFactory:try:if page and "html" in final_response.all_headers().get("content-type", ""):page_content = cls._get_page_content(page).encode("utf-8")+ encoding = "utf-8"else:page_content = final_response.body()except Exception as e: # pragma: no cover@@ -269,6 +270,7 @@ class ResponseFactory:try:if page and "html" in (await final_response.all_headers()).get("content-type", ""):page_content = (await cls._get_async_page_content(page)).encode("utf-8")+ encoding = "utf-8"else:page_content = await final_response.body()except Exception as e: # pragma: no coverdiff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.pyindex 1327db8..b422ead 100644--- a/tests/fetchers/test_response_handling.py+++ b/tests/fetchers/test_response_handling.py@@ -1,9 +1,57 @@-from unittest.mock import Mock+from unittest.mock import AsyncMock, Mock++import pytestfrom scrapling.parser import Selectorfrom scrapling.engines.toolbelt.convertor import ResponseFactory, Response+HTML_WITH_ACCENTED_TEXT = "<html><body>Bürger</body></html>"+LATIN9_CONTENT_TYPE = "text/html; charset=iso-8859-15"+++def make_sync_playwright_response():+ response = Mock()+ response.url = "https://example.com"+ response.status = 200+ response.status_text = "OK"+ response.headers = {"content-type": LATIN9_CONTENT_TYPE}+ response.all_headers = Mock(return_value={"content-type": LATIN9_CONTENT_TYPE})+ response.body = Mock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))+ response.request.all_headers = Mock(return_value={})+ response.request.redirected_from = None+ return response+++def make_sync_page():+ page = Mock()+ page.url = "https://example.com"+ page.content = Mock(return_value=HTML_WITH_ACCENTED_TEXT)+ page.context.cookies = Mock(return_value=[])+ return page+++def make_async_playwright_response():+ response = Mock()+ response.url = "https://example.com"+ response.status = 200+ response.status_text = "OK"+ response.headers = {"content-type": LATIN9_CONTENT_TYPE}+ response.all_headers = AsyncMock(return_value={"content-type": LATIN9_CONTENT_TYPE})+ response.body = AsyncMock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))+ response.request.all_headers = AsyncMock(return_value={})+ response.request.redirected_from = None+ return response+++def make_async_page():+ page = Mock()+ page.url = "https://example.com"+ page.content = AsyncMock(return_value=HTML_WITH_ACCENTED_TEXT)+ page.context.cookies = AsyncMock(return_value=[])+ return page++class TestResponseFactory:"""Test ResponseFactory functionality"""@@ -31,6 +79,72 @@ class TestResponseFactory:assert response.url == "https://example.com"assert isinstance(response, Response)+ def test_playwright_page_content_uses_utf8_encoding(self):+ """page.content() returns Unicode, so its encoded bytes are UTF-8."""+ mock_response = make_sync_playwright_response()+ mock_page = make_sync_page()++ response = ResponseFactory.from_playwright_response(+ mock_page,+ mock_response,+ mock_response,+ {"adaptive": False},+ collect_history=False,+ )++ assert response.encoding == "utf-8"+ assert "Bürger" in response.html_content+ assert "BÃ" not in response.html_content++ def test_playwright_raw_response_keeps_header_encoding(self):+ """Raw response bytes should still use the charset from Content-Type."""+ mock_response = make_sync_playwright_response()++ response = ResponseFactory.from_playwright_response(+ None,+ mock_response,+ mock_response,+ {"adaptive": False},+ collect_history=False,+ )++ assert response.encoding == "iso-8859-15"+ assert "Bürger" in response.html_content++ @pytest.mark.asyncio+ async def test_async_playwright_page_content_uses_utf8_encoding(self):+ """Async page.content() returns Unicode, so its encoded bytes are UTF-8."""+ mock_response = make_async_playwright_response()+ mock_page = make_async_page()++ response = await ResponseFactory.from_async_playwright_response(+ mock_page,+ mock_response,+ mock_response,+ {"adaptive": False},+ collect_history=False,+ )++ assert response.encoding == "utf-8"+ assert "Bürger" in response.html_content+ assert "BÃ" not in response.html_content++ @pytest.mark.asyncio+ async def test_async_playwright_raw_response_keeps_header_encoding(self):+ """Raw response bytes should still use the charset from Content-Type."""+ mock_response = make_async_playwright_response()++ response = await ResponseFactory.from_async_playwright_response(+ None,+ mock_response,+ mock_response,+ {"adaptive": False},+ collect_history=False,+ )++ assert response.encoding == "iso-8859-15"+ assert "Bürger" in response.html_content+def test_response_history_processing(self):"""Test processing response history"""# Mock responses with redirects