Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Fix Playwright page content encoding

Yehuda_LevS0ae508f72583
diff --git a/scrapling/engines/toolbelt/convertor.py b/scrapling/engines/toolbelt/convertor.py
index cf303c8..e875cf4 100644
--- a/scrapling/engines/toolbelt/convertor.py
+++ b/scrapling/engines/toolbelt/convertor.py
@@ -122,6 +122,7 @@ class ResponseFactory:
try:
if page and "html" in final_response.all_headers().get("content-type", ""):
page_content = cls._get_page_content(page).encode("utf-8")
+ encoding = "utf-8"
else:
page_content = final_response.body()
except Exception as e: # pragma: no cover
@@ -269,6 +270,7 @@ class ResponseFactory:
try:
if page and "html" in (await final_response.all_headers()).get("content-type", ""):
page_content = (await cls._get_async_page_content(page)).encode("utf-8")
+ encoding = "utf-8"
else:
page_content = await final_response.body()
except Exception as e: # pragma: no cover
diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py
index 1327db8..b422ead 100644
--- a/tests/fetchers/test_response_handling.py
+++ b/tests/fetchers/test_response_handling.py
@@ -1,9 +1,57 @@
-from unittest.mock import Mock
+from unittest.mock import AsyncMock, Mock
+
+import pytest
from scrapling.parser import Selector
from scrapling.engines.toolbelt.convertor import ResponseFactory, Response
+HTML_WITH_ACCENTED_TEXT = "<html><body>Bürger</body></html>"
+LATIN9_CONTENT_TYPE = "text/html; charset=iso-8859-15"
+
+
+def make_sync_playwright_response():
+ response = Mock()
+ response.url = "https://example.com"
+ response.status = 200
+ response.status_text = "OK"
+ response.headers = {"content-type": LATIN9_CONTENT_TYPE}
+ response.all_headers = Mock(return_value={"content-type": LATIN9_CONTENT_TYPE})
+ response.body = Mock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))
+ response.request.all_headers = Mock(return_value={})
+ response.request.redirected_from = None
+ return response
+
+
+def make_sync_page():
+ page = Mock()
+ page.url = "https://example.com"
+ page.content = Mock(return_value=HTML_WITH_ACCENTED_TEXT)
+ page.context.cookies = Mock(return_value=[])
+ return page
+
+
+def make_async_playwright_response():
+ response = Mock()
+ response.url = "https://example.com"
+ response.status = 200
+ response.status_text = "OK"
+ response.headers = {"content-type": LATIN9_CONTENT_TYPE}
+ response.all_headers = AsyncMock(return_value={"content-type": LATIN9_CONTENT_TYPE})
+ response.body = AsyncMock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))
+ response.request.all_headers = AsyncMock(return_value={})
+ response.request.redirected_from = None
+ return response
+
+
+def make_async_page():
+ page = Mock()
+ page.url = "https://example.com"
+ page.content = AsyncMock(return_value=HTML_WITH_ACCENTED_TEXT)
+ page.context.cookies = AsyncMock(return_value=[])
+ return page
+
+
class TestResponseFactory:
"""Test ResponseFactory functionality"""
@@ -31,6 +79,72 @@ class TestResponseFactory:
assert response.url == "https://example.com"
assert isinstance(response, Response)
+ def test_playwright_page_content_uses_utf8_encoding(self):
+ """page.content() returns Unicode, so its encoded bytes are UTF-8."""
+ mock_response = make_sync_playwright_response()
+ mock_page = make_sync_page()
+
+ response = ResponseFactory.from_playwright_response(
+ mock_page,
+ mock_response,
+ mock_response,
+ {"adaptive": False},
+ collect_history=False,
+ )
+
+ assert response.encoding == "utf-8"
+ assert "Bürger" in response.html_content
+ assert "BÃ" not in response.html_content
+
+ def test_playwright_raw_response_keeps_header_encoding(self):
+ """Raw response bytes should still use the charset from Content-Type."""
+ mock_response = make_sync_playwright_response()
+
+ response = ResponseFactory.from_playwright_response(
+ None,
+ mock_response,
+ mock_response,
+ {"adaptive": False},
+ collect_history=False,
+ )
+
+ assert response.encoding == "iso-8859-15"
+ assert "Bürger" in response.html_content
+
+ @pytest.mark.asyncio
+ async def test_async_playwright_page_content_uses_utf8_encoding(self):
+ """Async page.content() returns Unicode, so its encoded bytes are UTF-8."""
+ mock_response = make_async_playwright_response()
+ mock_page = make_async_page()
+
+ response = await ResponseFactory.from_async_playwright_response(
+ mock_page,
+ mock_response,
+ mock_response,
+ {"adaptive": False},
+ collect_history=False,
+ )
+
+ assert response.encoding == "utf-8"
+ assert "Bürger" in response.html_content
+ assert "BÃ" not in response.html_content
+
+ @pytest.mark.asyncio
+ async def test_async_playwright_raw_response_keeps_header_encoding(self):
+ """Raw response bytes should still use the charset from Content-Type."""
+ mock_response = make_async_playwright_response()
+
+ response = await ResponseFactory.from_async_playwright_response(
+ None,
+ mock_response,
+ mock_response,
+ {"adaptive": False},
+ collect_history=False,
+ )
+
+ assert response.encoding == "iso-8859-15"
+ assert "Bürger" in response.html_content
+
def test_response_history_processing(self):
"""Test processing response history"""
# Mock responses with redirects