Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: Add coverage for the new parts

Karim shoair70b1916777ba
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index a61aa07..4cc5e3b 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -740,7 +740,7 @@ class TestModeSplitContract:
assert result.structured_content is not None
assert result.structured_content["session_type"] == "stealthy"
session = _FakeStealthySession.instances[-1]
- assert session.kwargs == {**options, "block_ads": True}
+ assert session.kwargs == {**options, "block_ads": True, "record_requests": True}
assert server._sessions["browser"].session is session
closed = await client.call_tool("close_session", {"session_id": "browser"})
assert not closed.is_error
@@ -980,7 +980,7 @@ class TestMCPServerAuthentication:
monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)
built = ScraplingMCPServer(auth_token=SHARED_KEY)._build_server("0.0.0.0", 8000)
- assert len(built._tool_manager.list_tools()) == 14
+ assert len(built._tool_manager.list_tools()) == 16
def test_http_without_a_token_refuses_to_serve(self, monkeypatch):
"""The streamable-http transport requires authentication unless the caller explicitly opts out"""
@@ -1037,7 +1037,7 @@ class TestServerToolRegistration:
@pytest.mark.asyncio
async def test_tools_are_listed_with_expected_schemas(self):
- """All 14 tools are advertised, with plain content for screenshots, snapshots, actions, and JavaScript"""
+ """All 16 tools are advertised, with plain content for screenshots, snapshots, actions, and JavaScript"""
server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)
async with Client(server) as client:
assert client.instructions
@@ -1067,7 +1067,7 @@ class TestServerToolRegistration:
result = await client.call_tool(name, {})
assert result.is_error
- assert len(tools) == 14
+ assert len(tools) == 16
assert tools["browser_screenshot"].output_schema is None
assert tools["browser_snapshot"].output_schema is None
assert tools["browser_actions"].output_schema is None
@@ -1175,7 +1175,7 @@ class TestServerToolRegistration:
assert result.ttl_ms == 3_600_000 and result.cache_scope == "public"
annotations = {tool.name: tool.annotations for tool in result.tools if tool.annotations is not None}
- assert len(annotations) == 14
+ assert len(annotations) == 16
for name in (
"make_request",
"bulk_get",
diff --git a/tests/ai/test_browser_network.py b/tests/ai/test_browser_network.py
new file mode 100644
index 0000000..712a771
--- /dev/null
+++ b/tests/ai/test_browser_network.py
@@ -0,0 +1,1024 @@
+from json import loads
+from os import getenv
+from asyncio import Event, create_task, sleep, wait_for
+from dataclasses import replace
+from types import SimpleNamespace
+from urllib.parse import urlsplit
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+from mcp.client import Client
+from mcp.types import CallToolResult, TextContent
+
+from scrapling.core._types import Any
+from scrapling.core._network_formatting import NetworkRequestModel, _request_details
+from scrapling.core.ai import NetworkRequestsModel, ScraplingMCPServer, _SessionEntry
+from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest
+from scrapling.engines.toolbelt.custom import Response
+
+
+def _server() -> tuple[ScraplingMCPServer, Mock]:
+ server = ScraplingMCPServer()
+ saved = Response(
+ url="https://network.test/api/items",
+ content='{"name":"مرحبا"}'.encode(),
+ status=201,
+ reason="Created",
+ cookies={},
+ headers={"content-type": "application/json; charset=utf-8", "set-cookie": "test=2"},
+ request_headers={"content-type": "application/json", "cookie": "test=1"},
+ )
+ record = NetworkRequest(
+ id=4,
+ url=saved.url,
+ method="POST",
+ resource_type="fetch",
+ status=201,
+ request_headers=saved.request_headers,
+ request_body=b'{"sent":true}',
+ response_headers=saved.headers,
+ response=saved,
+ )
+ network = Mock(spec=["last_id", "dropped_count", "search", "get"], last_id=4, dropped_count=0)
+ network.search.return_value = [record]
+ network.get.side_effect = lambda request_id: record if request_id == 4 else None
+ session = SimpleNamespace(_is_alive=True, network=network)
+ server._sessions["browser"] = _SessionEntry(session, "stealthy")
+ return server, network
+
+
+def _recording_server(max_requests: int = 3) -> tuple[ScraplingMCPServer, NetworkRecorder, Mock]:
+ server, network, context = ScraplingMCPServer(), NetworkRecorder(True, max_requests, asynchronous=True), Mock()
+ network._attach(context)
+ session = SimpleNamespace(_is_alive=True, network=network)
+ server._sessions["browser"] = _SessionEntry(session, "stealthy")
+ return server, network, context
+
+
+def _native_request(path: str, resource_type: str = "fetch", status: int = 200) -> Mock:
+ request = Mock(
+ url=f"https://network.test{path}",
+ method="GET",
+ resource_type=resource_type,
+ headers={},
+ post_data_buffer=None,
+ )
+ request.all_headers = AsyncMock(return_value={})
+ response = Mock(
+ url=request.url,
+ status=status,
+ status_text="OK",
+ request=request,
+ headers={"content-type": "text/plain"},
+ )
+ response.all_headers = AsyncMock(return_value=response.headers.copy())
+ response.body = AsyncMock(return_value=b"saved")
+ request.existing_response = response
+ return request
+
+
+def _listing(result: CallToolResult) -> dict[str, Any]:
+ assert not result.is_error and isinstance(result.structured_content, dict)
+ assert len(result.content) == 1 and isinstance(result.content[0], TextContent)
+ assert loads(result.content[0].text) == result.structured_content
+ assert set(result.structured_content) == {"requests", "next_cursor", "has_more", "dropped_count"}
+ return result.structured_content
+
+
+def _detail(result: CallToolResult) -> dict[str, Any]:
+ assert not result.is_error and isinstance(result.structured_content, dict)
+ assert len(result.content) == 1 and isinstance(result.content[0], TextContent)
+ assert loads(result.content[0].text) == result.structured_content
+ assert set(result.structured_content) == {"request_id", "part", "data", "note"}
+ return result.structured_content
+
+
+def _detail_info(
+ request_id: int,
+ part: str,
+ data: Any = None,
+ note: str | None = None,
+) -> dict[str, Any]:
+ return {
+ "request_id": request_id,
+ "part": part,
+ "data": data,
+ "note": note,
+ }
+
+
+def _request_info(
+ request_id: int, path: str, method: str = "GET", resource_type: str = "fetch", status: int = 200
+) -> dict[str, Any]:
+ return {
+ "id": request_id,
+ "url": f"https://network.test{path}",
+ "method": method,
+ "resource_type": resource_type,
+ "status": status,
+ }
+
+
+async def test_network_tools_schema_and_content() -> None:
+ server, network = _server()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ tools = {tool.name: tool for tool in (await client.list_tools()).tools}
+ for name in ("browser_network_requests", "browser_network_request"):
+ tool = tools[name]
+ assert tool.annotations is not None and tool.annotations.read_only_hint is True
+ assert tool.annotations.open_world_hint is False
+ listing = tools["browser_network_requests"]
+ assert listing.title == "List network requests"
+ assert listing.input_schema["required"] == ["session_id"]
+ properties = listing.input_schema["properties"]
+ assert properties["include_static"]["default"] is False
+ assert properties["limit"]["default"] == 50 and properties["limit"]["maximum"] == 200
+ assert properties["after_id"]["minimum"] == 0
+ schema = listing.output_schema
+ assert schema is not None and schema["type"] == "object"
+ output_properties = schema["properties"]
+ assert {name: value["type"] for name, value in output_properties.items()} == {
+ "requests": "array",
+ "next_cursor": "integer",
+ "has_more": "boolean",
+ "dropped_count": "integer",
+ }
+ item_schema = schema["$defs"][output_properties["requests"]["items"]["$ref"].rsplit("/", 1)[-1]]
+ assert item_schema["type"] == "object"
+ assert {name: value["type"] for name, value in item_schema["properties"].items()} == {
+ "id": "integer",
+ "url": "string",
+ "method": "string",
+ "resource_type": "string",
+ "status": "integer",
+ }
+ detail = tools["browser_network_request"]
+ detail_schema = detail.output_schema
+ assert detail_schema is not None and detail_schema["type"] == "object"
+ output_properties = detail_schema["properties"]
+ assert set(output_properties) == {"request_id", "part", "data", "note"}
+ assert output_properties["request_id"]["type"] == "integer"
+ assert output_properties["part"]["type"] == "string"
+ assert set(output_properties["part"]["enum"]) == {
+ "summary",
+ "request_headers",
+ "request_body",
+ "response_headers",
+ "response_body",
+ }
+ assert {option["type"] for option in output_properties["note"]["anyOf"]} == {"string", "null"}
+ data_options = output_properties["data"]["anyOf"]
+ assert len(data_options) == 4
+ assert {option["type"] for option in data_options if "type" in option} == {"object", "string", "null"}
+ header_schema = next(option for option in data_options if option.get("type") == "object")
+ assert header_schema["additionalProperties"] == {"type": "string"}
+ summary_ref = next(option["$ref"] for option in data_options if "$ref" in option)
+ assert detail_schema["$defs"][summary_ref.rsplit("/", 1)[-1]] == item_schema
+ assert detail.title == "Read network request"
+ assert detail.input_schema["required"] == ["session_id", "request_id"]
+ properties = detail.input_schema["properties"]
+ assert properties["part"]["default"] == "summary"
+ assert set(properties["part"]["enum"]) == {
+ "summary",
+ "request_headers",
+ "request_body",
+ "response_headers",
+ "response_body",
+ }
+ assert set(properties) == {"session_id", "request_id", "part"}
+ assert properties["request_id"]["minimum"] == 1
+ result = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ assert _listing(result) == {
+ "requests": [_request_info(4, "/api/items", "POST", status=201)],
+ "next_cursor": 4,
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ network.search.assert_called_once_with(url_pattern=None, include_static=False, after_id=0, limit=51)
+
+
+async def test_network_list_forwards_filters_and_reports_paging_eviction() -> None:
+ server, network = _server()
+ network.search.return_value.append(
+ replace(
+ network.get(4),
+ id=8,
+ url="https://network.test/next",
+ method="GET",
+ resource_type="document",
+ )
+ )
+ network.last_id = 8
+ network.dropped_count = 3
+ result = await server.browser_network_requests(
+ "browser", url_pattern="/api/", include_static=True, after_id=2, limit=1
+ )
+ network.search.assert_called_once_with(url_pattern="/api/", include_static=True, after_id=2, limit=2)
+ assert isinstance(result, NetworkRequestsModel)
+ assert result.model_dump() == {
+ "requests": [_request_info(4, "/api/items", "POST", status=201)],
+ "next_cursor": 4,
+ "has_more": True,
+ "dropped_count": 3,
+ }
+ network.search.return_value = []
+ result = await server.browser_network_requests("browser", after_id=10)
+ assert result.model_dump() == {"requests": [], "next_cursor": 10, "has_more": False, "dropped_count": 3}
+
+
+async def test_network_cursor_follows_save_order_and_survives_clear() -> None:
+ server, network, context = _recording_server()
+ callback = network._contexts[context]["requestfinished"]
+ started, release = Event(), Event()
+ slow, fast = _native_request("/api/slow"), _native_request("/api/fast")
+
+ async def slow_body() -> bytes:
+ started.set()
+ await release.wait()
+ return b"slow"
+
+ slow.existing_response.body.side_effect = slow_body
+ capture = create_task(callback(slow))
+ try:
+ await wait_for(started.wait(), timeout=2)
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ empty = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ assert _listing(empty) == {"requests": [], "next_cursor": 0, "has_more": False, "dropped_count": 0}
+ await callback(fast)
+ first = await client.call_tool("browser_network_requests", {"session_id": "browser", "limit": 1})
+ assert _listing(first) == {
+ "requests": [_request_info(1, "/api/fast")],
+ "next_cursor": 1,
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ release.set()
+ await wait_for(capture, timeout=2)
+ second = await client.call_tool(
+ "browser_network_requests", {"session_id": "browser", "after_id": 1, "limit": 1}
+ )
+ assert _listing(second) == {
+ "requests": [_request_info(2, "/api/slow")],
+ "next_cursor": 2,
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ network.clear()
+ await callback(_native_request("/api/new"))
+ after_clear = await client.call_tool("browser_network_requests", {"session_id": "browser", "after_id": 2})
+ assert _listing(after_clear) == {
+ "requests": [_request_info(3, "/api/new")],
+ "next_cursor": 3,
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ assert network.get(2) is None
+ finally:
+ release.set()
+ await wait_for(capture, timeout=2)
+ network._detach()
+
+
+async def test_network_real_filtering_eviction_and_cursor_paging() -> None:
+ server, network, context = _recording_server()
+ callback = network._contexts[context]["requestfinished"]
+ try:
+ for native in (
+ _native_request("/page", "document"),
+ _native_request("/api/first"),
+ _native_request("/missing.png", "image", 404),
+ ):
+ await callback(native)
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ first = await client.call_tool("browser_network_requests", {"session_id": "browser", "limit": 1})
+ assert _listing(first) == {
+ "requests": [_request_info(2, "/api/first")],
+ "next_cursor": 2,
+ "has_more": True,
+ "dropped_count": 0,
+ }
+ await callback(_native_request("/site.css", "stylesheet"))
+ await callback(_native_request("/api/last", status=503))
+ next_page = await client.call_tool("browser_network_requests", {"session_id": "browser", "after_id": 2})
+ assert _listing(next_page) == {
+ "requests": [
+ _request_info(3, "/missing.png", resource_type="image", status=404),
+ _request_info(5, "/api/last", status=503),
+ ],
+ "next_cursor": 5,
+ "has_more": False,
+ "dropped_count": 2,
+ }
+ all_types = await client.call_tool(
+ "browser_network_requests", {"session_id": "browser", "include_static": True, "after_id": 2}
+ )
+ assert _listing(all_types) == {
+ "requests": [
+ _request_info(3, "/missing.png", resource_type="image", status=404),
+ _request_info(4, "/site.css", resource_type="stylesheet"),
+ _request_info(5, "/api/last", status=503),
+ ],
+ "next_cursor": 5,
+ "has_more": False,
+ "dropped_count": 2,
+ }
+ filtered = await client.call_tool(
+ "browser_network_requests", {"session_id": "browser", "url_pattern": r"/api/"}
+ )
+ assert _listing(filtered) == {
+ "requests": [_request_info(5, "/api/last", status=503)],
+ "next_cursor": 5,
+ "has_more": False,
+ "dropped_count": 2,
+ }
+ evicted = await client.call_tool("browser_network_request", {"session_id": "browser", "request_id": 2})
+ assert isinstance(evicted.content[0], TextContent)
+ assert evicted.is_error and "no longer retained" in evicted.content[0].text
+ invalid = await client.call_tool("browser_network_requests", {"session_id": "browser", "url_pattern": "["})
+ assert isinstance(invalid.content[0], TextContent)
+ assert invalid.is_error and "Invalid URL pattern" in invalid.content[0].text
+ finally:
+ network._detach()
+
+
[email protected]("count,after_id,next_cursor", [(0, 0, 0), (2, 0, 2), (2, 1, 2), (2, 2, 2), (2, 9, 9)])
+async def test_network_empty_matches_keep_cursor_at_or_after_saved_history(
+ count: int, after_id: int, next_cursor: int
+) -> None:
+ server, network, context = _recording_server()
+ try:
+ for index in range(count):
+ await network._contexts[context]["requestfinished"](_native_request(f"/api/{index}"))
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(
+ "browser_network_requests",
+ {"session_id": "browser", "url_pattern": "/missing/", "after_id": after_id},
+ )
+ assert _listing(result) == {
+ "requests": [],
+ "next_cursor": next_cursor,
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ assert network.last_id == count
+ finally:
+ network._detach()
+
+
[email protected]("count,has_more", [(2, False), (3, True)])
+async def test_network_listing_limit_checks_for_an_extra_match(count: int, has_more: bool) -> None:
+ server, network, context = _recording_server()
+ try:
+ for index in range(1, count + 1):
+ await network._contexts[context]["requestfinished"](_native_request(f"/api/{index}"))
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool("browser_network_requests", {"session_id": "browser", "limit": 2})
+ assert _listing(result) == {
+ "requests": [_request_info(1, "/api/1"), _request_info(2, "/api/2")],
+ "next_cursor": 2,
+ "has_more": has_more,
+ "dropped_count": 0,
+ }
+ finally:
+ network._detach()
+
+
+async def test_network_listing_preserves_url_characters_and_only_returns_summary_fields() -> None:
+ server, network, context = _recording_server()
+ path = '/api/مرحبا?q="quoted"&path=one\\two#one\ntwo'
+ native = _native_request(path)
+ native.method = "POST"
+ native.post_data_buffer = b'{"secret":"request"}'
+ native.headers = {"content-type": "application/json", "cookie": "secret=request"}
+ native.all_headers.return_value = native.headers.copy()
+ native.existing_response.headers = {"content-type": "application/json", "set-cookie": "secret=response"}
+ native.existing_response.all_headers.return_value = native.existing_response.headers.copy()
+ native.existing_response.body.return_value = b'{"secret":"response"}'
+ await network._contexts[context]["requestfinished"](native)
+ network._detach()
+ record = network.get(1)
+ assert record is not None and record.url == native.url
+ assert record.request_body == b'{"secret":"request"}' and record.response.body == b'{"secret":"response"}'
+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
+ for reader in readers:
+ reader.assert_awaited_once()
+ reader.side_effect = AssertionError("Listing saved history must not read the browser")
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ assert _listing(result) == {
+ "requests": [_request_info(1, path, "POST")],
+ "next_cursor": 1,
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ for reader in readers:
+ reader.assert_awaited_once()
+
+
[email protected]("part", ["summary", "request_headers", "request_body", "response_headers", "response_body"])
+async def test_network_detail_returns_only_selected_part(part: Any) -> None:
+ server, network = _server()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 4, "part": part}
+ )
+ record = network.get(4)
+ if part == "summary":
+ assert _detail(result) == _detail_info(4, part, _request_info(4, "/api/items", "POST", status=201))
+ elif part.endswith("headers"):
+ expected = record.response.request_headers if part == "request_headers" else record.response.headers
+ assert _detail(result) == _detail_info(4, part, expected)
+ else:
+ expected = '{"sent":true}' if part == "request_body" else '{"name":"مرحبا"}'
+ assert _detail(result) == _detail_info(4, part, expected)
+ assert not hasattr(record, "request")
+ network.search.assert_not_called()
+
+
[email protected]("part", ["request_headers", "response_headers", "response_body"])
+def test_saved_response_details_are_local(part: Any) -> None:
+ _, network = _server()
+ record = network.get(4)
+ result = _request_details(record, part)
+ assert isinstance(result, NetworkRequestModel)
+ if part.endswith("headers"):
+ expected = record.response.request_headers if part == "request_headers" else record.response.headers
+ assert result.model_dump() == _detail_info(4, part, expected)
+ else:
+ assert result.model_dump() == _detail_info(4, part, '{"name":"مرحبا"}')
+
+
[email protected]("part", ["summary", "request_headers", "response_headers"])
+async def test_network_objects_keep_all_summary_fields_and_header_values(part: str) -> None:
+ server, network = _server()
+ record = network.get(4)
+ headers = {
+ "id": "007",
+ "url": "https://headers.test/",
+ "method": "PATCH",
+ "resource_type": "xhr",
+ "status": "0200",
+ "x-extra": "kept",
+ }
+ if part == "request_headers":
+ record.request_headers = headers
+ elif part == "response_headers":
+ record.response_headers = headers
+ expected = _request_info(4, "/api/items", "POST", status=201) if part == "summary" else headers
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(
+ "browser_network_request",
+ {"session_id": "browser", "request_id": 4, "part": part},
+ )
+ assert _detail(result) == _detail_info(4, part, expected)
+
+
[email protected]("part", ["request_body", "response_body"])
+async def test_network_unicode_body_returns_the_full_original_text(part: str) -> None:
+ server, network = _server()
+ record = network.get(4)
+ text = 'Aمرحبا🙂e\u0301\n"z"'
+ if part == "request_body":
+ record.request_body = text.encode()
+ else:
+ record.response._raw_body = text.encode()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 4, "part": part}
+ )
+ assert _detail(result) == _detail_info(4, part, text)
+
+
[email protected]("part", ["request_body", "response_body"])
[email protected]("size", [0, 10001, 50001, 100003])
+async def test_network_detail_returns_complete_empty_and_large_saved_bodies(part: str, size: int) -> None:
+ server, network, context = _recording_server()
+ native = _native_request("/api/full-body")
+ text = 'start\n"' + "x" * (size - 11) + '"end' if size else ""
+ assert len(text) == size
+ if part == "request_body":
+ native.post_data_buffer = text.encode()
+ native.headers = {"content-type": "text/plain"}
+ native.all_headers.return_value = native.headers.copy()
+ else:
+ native.existing_response.body.return_value = text.encode()
+ await network._contexts[context]["requestfinished"](native)
+ network._detach()
+ record = network.get(1)
+ assert record is not None
+ assert (record.request_body if part == "request_body" else record.response.body) == text.encode()
+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
+ for reader in readers:
+ reader.assert_awaited_once()
+ reader.side_effect = AssertionError("Reading full saved bodies must not read the browser")
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}
+ )
+ assert _detail(result) == _detail_info(1, part, text)
+ for reader in readers:
+ reader.assert_awaited_once()
+
+
[email protected]("case", ["empty", "binary", "unavailable", "too_large"])
+async def test_network_detail_body_availability(case: str) -> None:
+ server, network = _server()
+ record = network.get(4)
+ if case == "empty":
+ record.response.status = 204
+ record.response._raw_body = b""
+ elif case == "binary":
+ record.response.headers["content-type"] = "image/png"
+ elif case == "too_large":
+ record.response.meta["body_note"] = "too large; not saved."
+ else:
+ record.response.meta["body_note"] = "Context closed"
+ result = await server.browser_network_request("browser", 4, "response_body")
+ expected = {
+ "empty": _detail_info(4, "response_body", ""),
+ "binary": _detail_info(4, "response_body", note="Only text bodies can be displayed."),
+ "unavailable": _detail_info(4, "response_body", note="Context closed"),
+ "too_large": _detail_info(4, "response_body", note="too large; not saved."),
+ }
+ assert result.model_dump() == expected[case]
+
+
[email protected]("part", ["request_headers", "response_headers"])
[email protected]("partial", [False, True])
+async def test_network_detail_saved_headers_report_completeness(part: Any, partial: bool) -> None:
+ server, network = _server()
+ record = network.get(4)
+ headers = record.request_headers if part == "request_headers" else record.response_headers
+ key = "request_headers_partial" if part == "request_headers" else "headers_partial"
+ record.response.meta[key] = partial
+ result = await server.browser_network_request("browser", 4, part)
+ assert result.model_dump() == _detail_info(
+ 4, part, headers, note="Headers are partial; full headers unavailable." if partial else None
+ )
+
+
+ "content_type, body, encoding, expected",
+ [
+ ("text/plain; charset = iso-8859-1", b"caf\xe9", "iso-8859-1", "café"),
+ ("text/plain; charset=invalid-codec", b"hi", "invalid-codec", "hi"),
+ ('text/plain; CHARSET = "ISO-8859-1"', b"caf\xe9", "iso-8859-1", "café"),
+ ("text/plain; charset = 'ISO-8859-1'", b"caf\xe9", "iso-8859-1", "café"),
+ ("text/plain; charset=iso-8859-1", "café".encode(), "utf-8", "café"),
+ ("application/problem+json", b'{"error":true}', "utf-8", '{"error":true}'),
+ ("application/xml", b"<ok/>", "utf-8", "<ok/>"),
+ ],
+)
+async def test_network_detail_decodes_text_for_any_resource(
+ content_type: str, body: bytes, encoding: str, expected: str
+) -> None:
+ server, network = _server()
+ record = network.get(4)
+ record.resource_type = "document"
+ record.response.headers["content-type"] = content_type
+ record.response._raw_body = body
+ record.response.encoding = encoding
+ result = await server.browser_network_request("browser", 4, "response_body")
+ assert result.model_dump() == _detail_info(4, "response_body", expected)
+
+
+ "response_type,native_body,response_encoding,response_text,request_type,request_body,request_text",
+ [
+ (
+ "text/plain; charset=iso-8859-15",
+ "café ¤".encode(),
+ "utf-8",
+ "café ¤",
+ "text/plain; charset=iso-8859-15",
+ b"caf\xe9 \xa4",
+ "café €",
+ ),
+ (
+ "text/plain; charset=iso-8859-1",
+ b"caf\xe9",
+ "iso-8859-1",
+ "café",
+ "text/plain; charset=iso-8859-1",
+ b"caf\xe9",
+ "café",
+ ),
+ (
+ "text/plain; charset=invalid-codec",
+ "café".encode(),
+ "utf-8",
+ "café",
+ "text/plain; charset=invalid-codec",
+ "café".encode(),
+ "café",
+ ),
+ (
+ "text/plain; charset=invalid-codec",
+ b"caf\xe9",
+ "utf-8",
+ "caf\ufffd",
+ "text/plain; charset=invalid-codec",
+ b"caf\xe9",
+ "caf\ufffd",
+ ),
+ ],
+)
+async def test_network_response_uses_saved_encoding_and_request_uses_its_header(
+ response_type: str,
+ native_body: bytes,
+ response_encoding: str,
+ response_text: str,
+ request_type: str,
+ request_body: bytes,
+ request_text: str,
+) -> None:
+ server, network, context = _recording_server()
+ native = _native_request("/api/encoded")
+ native.method = "POST"
+ native.headers = {"content-type": request_type}
+ native.all_headers.return_value = native.headers.copy()
+ native.post_data_buffer = request_body
+ native.existing_response.headers = {"content-type": response_type}
+ native.existing_response.all_headers.return_value = native.existing_response.headers.copy()
+ native.existing_response.body.return_value = native_body
+ await network._contexts[context]["requestfinished"](native)
+ network._detach()
+ record = network.get(1)
+ assert record is not None
+ assert record.response.encoding == response_encoding
+ assert record.response.body == native_body and record.request_body == request_body
+ assert record.response_headers == {"content-type": response_type}
+ assert record.request_headers == {"content-type": request_type}
+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
+ for reader in readers:
+ reader.assert_awaited_once()
+ reader.side_effect = AssertionError("Saved history must not read the browser")
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ for part, expected in (("response_body", response_text), ("request_body", request_text)):
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}
+ )
+ assert _detail(result) == _detail_info(1, part, expected)
+ for part, expected in (("response_headers", response_type), ("request_headers", request_type)):
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}
+ )
+ assert _detail(result) == _detail_info(1, part, {"content-type": expected})
+ for reader in readers:
+ reader.assert_awaited_once()
+ assert record.response.body == native_body and record.request_body == request_body
+
+
+async def test_network_request_body_absent_and_binary() -> None:
+ server, network = _server()
+ record = network.get(4)
+ record.request_body = None
+ result = await server.browser_network_request("browser", 4, "request_body")
+ assert result.model_dump() == _detail_info(4, "request_body", note="Body is absent.")
+ record.request_body = b"binary"
+ record.request_headers["content-type"] = "application/octet-stream"
+ record.response.request_headers["content-type"] = "application/octet-stream"
+ result = await server.browser_network_request("browser", 4, "request_body")
+ assert result.model_dump() == _detail_info(4, "request_body", note="Only text bodies can be displayed.")
+
+
+ "part,body,content_type,status,expected,note",
+ [
+ ("request_body", None, "text/plain", 200, None, "Body is absent."),
+ ("request_body", b"", "text/plain", 200, "", None),
+ ("request_body", b"", None, 200, "", None),
+ ("request_body", b"binary", "application/octet-stream", 200, None, "Only text bodies can be displayed."),
+ ("response_body", b"", "text/plain", 200, "", None),
+ ("response_body", b"", "application/json", 204, "", None),
+ ("response_body", b"", None, 200, None, "Non-text body; not saved."),
+ ("response_body", b"", None, 204, "", None),
+ ("response_body", b"binary", "image/png", 200, None, "Non-text body; not saved."),
+ ("response_body", b"file", "application/pdf", 200, None, "Non-text body; not saved."),
+ ("response_body", b"file", "application/octet-stream", 200, None, "Non-text body; not saved."),
+ ("response_body", b"saved", "application/unknown", 200, None, "Non-text body; not saved."),
+ ("response_body", b"saved", None, 200, None, "Non-text body; not saved."),
+ ("response_body", b'{"ok":true}', "application/problem+json", 200, '{"ok":true}', None),
+ ],
+)
+async def test_network_reads_saved_parts_without_native_access(
+ part: str, body: bytes | None, content_type: str | None, status: int, expected: str | None, note: str | None
+) -> None:
+ server, network, context = _recording_server()
+ native = _native_request("/api/saved", status=status)
+ headers = {"content-type": content_type} if content_type is not None else {}
+ if part == "request_body":
+ native.post_data_buffer = body
+ native.headers = headers
+ native.all_headers.return_value = headers.copy()
+ else:
+ native.existing_response.body.return_value = body
+ native.existing_response.headers = headers
+ native.existing_response.all_headers.return_value = headers.copy()
+ await network._contexts[context]["requestfinished"](native)
+ network._detach()
+ record = network.get(1)
+ assert record is not None
+ skipped = part == "response_body" and note == "Non-text body; not saved."
+ assert record.response.meta == ({"body_note": note} if skipped else {})
+ if part == "response_body":
+ assert record.response.body == (b"" if skipped else body)
+ assert native.existing_response.body.await_count == (not skipped and status != 204)
+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
+ counts = [reader.await_count for reader in readers]
+ for reader in readers:
+ reader.side_effect = AssertionError("Saved history must not read the browser")
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}
+ )
+ assert _detail(result) == _detail_info(1, part, expected, note=note)
+ for saved_part in ("summary", "request_headers", "response_headers"):
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": saved_part}
+ )
+ expected_data = (
+ _request_info(1, "/api/saved", status=status)
+ if saved_part == "summary"
+ else record.request_headers
+ if saved_part == "request_headers"
+ else record.response_headers
+ )
+ assert _detail(result) == _detail_info(1, saved_part, expected_data)
+ assert [reader.await_count for reader in readers] == counts
+
+
+ "error,expected",
+ [
+ (None, "too large; not saved."),
+ (RuntimeError("Context closed"), "Context closed"),
+ (RuntimeError(), "could not be saved."),
+ ],
+)
+async def test_network_body_note_does_not_replace_other_saved_parts(error: Exception | None, expected: str) -> None:
+ server, network, context = _recording_server()
+ native = _native_request("/api/body-note")
+ native.method = "POST"
+ native.post_data_buffer = b'{"sent":true}'
+ native.headers = {"content-type": "application/json"}
+ native.all_headers.return_value = native.headers.copy()
+ response_headers = {"content-type": "application/json", "x-recorded": "yes"}
+ if error is None:
+ response_headers["content-length"] = str(1024 * 1024 + 1)
+ else:
+ native.existing_response.body.side_effect = error
+ native.existing_response.headers = response_headers
+ native.existing_response.all_headers.return_value = response_headers.copy()
+ await network._contexts[context]["requestfinished"](native)
+ network._detach()
+ record = network.get(1)
+ assert record is not None and record.response.meta == {"body_note": expected}
+ assert record.response.body == b"" and record.request_body == b'{"sent":true}'
+ assert native.existing_response.body.await_count == (error is not None)
+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
+ counts = [reader.await_count for reader in readers]
+ for reader in readers:
+ reader.side_effect = AssertionError("Saved history must not read the browser")
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ for part in ("response_body", "summary", "request_headers", "response_headers", "request_body"):
+ result = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}
+ )
+ if part == "response_body":
+ assert _detail(result) == _detail_info(1, part, note=expected)
+ elif part == "summary":
+ assert _detail(result) == _detail_info(1, part, _request_info(1, "/api/body-note", "POST"))
+ elif part.endswith("headers"):
+ assert _detail(result) == _detail_info(
+ 1, part, native.headers if part == "request_headers" else response_headers
+ )
+ else:
+ assert _detail(result) == _detail_info(1, part, '{"sent":true}')
+ assert [reader.await_count for reader in readers] == counts
+
+
+ "name, options",
+ [
+ ("browser_network_requests", {"after_id": -1}),
+ ("browser_network_requests", {"limit": 0}),
+ ("browser_network_requests", {"limit": 201}),
+ ("browser_network_request", {"request_id": 0}),
+ ("browser_network_request", {}),
+ ("browser_network_request", {"request_id": 4, "part": "all"}),
+ ],
+)
+async def test_network_invalid_input_does_not_read_history(name: str, options: dict[str, Any]) -> None:
+ server, network = _server()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool(name, {"session_id": "browser", **options})
+ assert result.is_error
+ network.get.assert_not_called()
+ network.search.assert_not_called()
+
+
[email protected]("case", ["missing", "static", "dead"])
[email protected]("name", ["browser_network_requests", "browser_network_request"])
+async def test_network_session_guards(case: str, name: str) -> None:
+ server, network = _server()
+ if case == "missing":
+ server._sessions.clear()
+ elif case == "static":
+ server._sessions["browser"].session_type = "static"
+ else:
+ server._sessions["browser"].session._is_alive = False
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ options = {"request_id": 4} if name == "browser_network_request" else {}
+ assert (await client.call_tool(name, {"session_id": "browser", **options})).is_error
+ network.get.assert_not_called()
+ network.search.assert_not_called()
+
+
+async def test_network_missing_id_and_search_error_reach_client() -> None:
+ server, network = _server()
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ result = await client.call_tool("browser_network_request", {"session_id": "browser", "request_id": 99})
+ assert isinstance(result.content[0], TextContent)
+ assert result.is_error and "no longer retained" in result.content[0].text
+ network.search.side_effect = ValueError("Invalid URL regular expression")
+ result = await client.call_tool("browser_network_requests", {"session_id": "browser", "url_pattern": "["})
+ assert isinstance(result.content[0], TextContent)
+ assert result.is_error and "regular expression" in result.content[0].text
+
+
+async def test_network_live_capture_actions_navigation_and_reads() -> None:
+ server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))
+ seen: list[str] = []
+ html = """<html><head><link rel="icon" href="data:,"></head><body><button id="load" onclick="fetch('/api/items', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({sent: true})}).then(r => r.json()).then(() => document.body.dataset.done='yes')">Load</button></body></html>"""
+ responses = {
+ "/api/items": (200, "application/json", b'{"items":[1,2]}'),
+ "/api/empty": (200, "text/plain", b""),
+ "/api/absent": (204, "application/json", b""),
+ "/api/binary": (200, "application/octet-stream", b"\x00\xff\x80binary"),
+ "/api/error": (503, "application/json", b'{"error":"unavailable"}'),
+ }
+
+ async def serve(route: Any) -> None:
+ seen.append(route.request.url)
+ path = urlsplit(route.request.url).path
+ if path == "/api/failed":
+ await route.abort("failed")
+ return
+ status, content_type, body = responses.get(path, (200, "text/html", html.encode()))
+ await route.fulfill(status=status, content_type=content_type, body=body, headers={"x-recorded": "yes"})
+
+ async def captured(session: Any, pattern: str) -> NetworkRequest:
+ while not (records := session.network.search(url_pattern=pattern)):
+ await sleep(0.01)
+ return records[0]
+
+ async with Client(server._build_server("127.0.0.1", 8000)) as client:
+ opened = await client.call_tool("browser_open", {"session_id": "browser"})
+ assert not opened.is_error
+ session = server._sessions["browser"].session
+ try:
+ assert session.network.enabled and opened.structured_content["settings"]["record_requests"] is True
+ empty = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ assert _listing(empty) == {"requests": [], "next_cursor": 0, "has_more": False, "dropped_count": 0}
+ await session.context.route("**/*", serve)
+ assert not (
+ await client.call_tool("browser_fetch", {"session_id": "browser", "url": "https://network.test/"})
+ ).is_error
+ acted = await client.call_tool(
+ "browser_actions",
+ {
+ "session_id": "browser",
+ "actions": [
+ {"type": "click", "selector": "#load"},
+ {"type": "wait_element", "selector": "body[data-done=yes]", "timeout": 2000},
+ ],
+ },
+ )
+ assert not acted.is_error
+ request = await wait_for(captured(session, "/api/items$"), timeout=5)
+ extra = await client.call_tool(
+ "browser_evaluate",
+ {
+ "session_id": "browser",
+ "expression": "async () => { await Promise.allSettled(['/api/empty', '/api/absent', '/api/binary', '/api/error', '/api/failed'].map(path => fetch(path).then(response => response.arrayBuffer()))); return true; }",
+ },
+ )
+ assert not extra.is_error
+ records = {
+ name: await wait_for(captured(session, f"/api/{name}$"), timeout=5)
+ for name in ("empty", "absent", "binary", "error")
+ }
+ assert session.network.search(url_pattern="/api/failed$") == []
+ for name, record in records.items():
+ assert record.response.body == (b"" if name == "binary" else responses[f"/api/{name}"][2])
+ assert record.response.meta == ({"body_note": "Non-text body; not saved."} if name == "binary" else {})
+ before = list(seen)
+ page = session.page_pool.get_ready_page()
+ assert page is not None
+ try:
+ listed = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ detail = await client.call_tool(
+ "browser_network_request",
+ {"session_id": "browser", "request_id": request.id, "part": "response_body"},
+ )
+ assert not listed.is_error and not detail.is_error
+ assert request.response is not None and request.response.body == b'{"items":[1,2]}'
+ expected_requests = sorted(
+ [_request_info(request.id, "/api/items", "POST")]
+ + [
+ _request_info(record.id, f"/api/{name}", status=responses[f"/api/{name}"][0])
+ for name, record in records.items()
+ ],
+ key=lambda item: item["id"],
+ )
+ expected_listing = {
+ "requests": expected_requests,
+ "next_cursor": expected_requests[-1]["id"],
+ "has_more": False,
+ "dropped_count": 0,
+ }
+ assert _listing(listed) == expected_listing
+ assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')
+ request_body = await client.call_tool(
+ "browser_network_request",
+ {"session_id": "browser", "request_id": request.id, "part": "request_body"},
+ )
+ assert _detail(request_body) == _detail_info(request.id, "request_body", '{"sent":true}')
+ for name, expected, note in (
+ ("empty", "", None),
+ ("absent", "", None),
+ ("binary", None, "Non-text body; not saved."),
+ ("error", '{"error":"unavailable"}', None),
+ ):
+ result = await client.call_tool(
+ "browser_network_request",
+ {"session_id": "browser", "request_id": records[name].id, "part": "response_body"},
+ )
+ assert _detail(result) == _detail_info(records[name].id, "response_body", expected, note=note)
+ headers = await client.call_tool(
+ "browser_network_request",
+ {"session_id": "browser", "request_id": records[name].id, "part": "response_headers"},
+ )
+ assert _detail(headers) == _detail_info(
+ records[name].id, "response_headers", records[name].response_headers
+ )
+ assert _detail(headers)["data"]["x-recorded"] == "yes"
+ assert page.state == "busy" and seen == before
+ finally:
+ page.mark_ready()
+ assert not (
+ await client.call_tool("browser_fetch", {"session_id": "browser", "url": "https://network.test/next"})
+ ).is_error
+ await wait_for(captured(session, "/next$"), timeout=5)
+ listed = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ assert _listing(listed) == expected_listing
+ await session.close_pages()
+ before = list(seen)
+ detail = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}
+ )
+ assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')
+ for record in records.values():
+ saved = await client.call_tool(
+ "browser_network_request",
+ {"session_id": "browser", "request_id": record.id, "part": "response_headers"},
+ )
+ assert _detail(saved) == _detail_info(record.id, "response_headers", record.response_headers)
+ assert _detail(saved)["data"]["x-recorded"] == "yes"
+ assert seen == before
+ finally:
+ try:
+ assert not (await client.call_tool("close_session", {"session_id": "browser"})).is_error
+ finally:
+ await session.close()
+ assert not session.network._contexts
+ assert request.response.body == b'{"items":[1,2]}'
+ closed = await client.call_tool("browser_network_requests", {"session_id": "browser"})
+ assert closed.is_error
+ closed_detail = await client.call_tool(
+ "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}
+ )
+ assert closed_detail.is_error
diff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.py
new file mode 100644
index 0000000..45dca2a
--- /dev/null
+++ b/tests/fetchers/test_network_browser.py
@@ -0,0 +1,952 @@
+import asyncio
+from base64 import b64decode
+from contextlib import suppress
+from dataclasses import dataclass, field
+from gzip import compress
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from json import loads
+from os import getenv
+from socket import SHUT_RDWR
+from threading import Event, Thread
+from time import monotonic
+from unittest.mock import Mock
+from urllib.parse import urlsplit
+
+import pytest
+from patchright.async_api import Response as AsyncPatchrightResponse
+from patchright.sync_api import Response as PatchrightResponse
+from playwright.async_api import Response as AsyncPlaywrightResponse
+from playwright.sync_api import Response as PlaywrightResponse
+
+from scrapling.core._types import Any, Generator
+from scrapling.core._network_formatting import _request_details
+from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
+
+
+HTML = b"""<!DOCTYPE html><html><head><link rel="icon" href="data:,"><link rel="stylesheet" href="/style.css"></head>
+<body><img src="/image.png"><script src="/script.js"></script></body></html>"""
+SCRIPT = b"""Promise.all([
+fetch('/api/initial').then(response => response.json()),
+new Promise(resolve => { const xhr = new XMLHttpRequest(); xhr.open('GET', '/api/xhr');
+xhr.onload = () => resolve(JSON.parse(xhr.responseText)); xhr.send(); })
+]).then(() => document.body.dataset.initial = 'ready');"""
+LATER = """async () => {
+ await fetch('/api/post', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-Sent': 'request'}, body: '{"sent":true}'}).then(response => response.json());
+ await fetch('/api/repeated').then(response => response.text());
+ await fetch('/api/repeated').then(response => response.text());
+ await fetch('/api/status').then(response => response.text());
+ await fetch('/redirect').then(response => response.text());
+ await fetch('/api/fail').catch(() => null);
+ await fetch('/api/large').then(response => response.text());
+}"""
+PENDING = """() => {
+ fetch('/api/pending').then(response => response.text()).then(() => document.body.dataset.pending = 'done');
+}"""
+BLANK = b'<!DOCTYPE html><html><head><link rel="icon" href="data:,"></head><body>ready</body></html>'
+LARGE = b"x" * (1024 * 1024 + 1)
+BINARY = bytes(range(256))
+PNG = b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aZ1cAAAAASUVORK5CYII=")
+TEXT_RESPONSES = {
+ "/mime/json": ("application/json", b'{"saved":true}'),
+ "/mime/xml": ("application/xml", b"<saved>true</saved>"),
+ "/mime/vendor-json": ("application/vnd.scrapling+json", b'{"saved":true}'),
+ "/mime/vendor-xml": ("application/vnd.scrapling+xml", b"<saved>true</saved>"),
+ "/mime/javascript": ("application/javascript", b"window.saved = true;"),
+ "/mime/graphql": ("application/graphql", b"{ saved }"),
+ "/mime/form": ("application/x-www-form-urlencoded", b"saved=true"),
+}
+MIME_RESPONSES = {
+ **TEXT_RESPONSES,
+ "/mime/image": ("image/png", PNG),
+ "/mime/pdf": ("application/pdf", b"%PDF-1.7\n%%EOF"),
+ "/mime/file": ("application/octet-stream", BINARY),
+ "/mime/media": ("audio/wav", b"RIFF\x04\x00\x00\x00WAVE"),
+ "/mime/font": ("font/woff2", b"wOF2\x00\x01\x00\x00"),
+ "/mime/missing": (None, b'{"saved":false}'),
+ "/mime/unknown": ("application/x-scrapling-unknown", b'{"saved":false}'),
+}
+MIME_FETCH = """async paths => {
+ const bodies = {};
+ for (const path of paths) {
+ const response = await fetch(path);
+ bodies[path] = Array.from(new Uint8Array(await response.arrayBuffer()));
+ }
+ return bodies;
+}"""
+CHARSET_BODY = "caf\u00e9 \u20ac".encode("iso-8859-15")
+CHARSET_PATHS = ("/wire/charset", "/wire/charset-spaced", "/wire/raw-charset")
+WIRE_PATHS = ["/wire/chunked", "/wire/gzip", "/wire/binary", *CHARSET_PATHS, "/wire/head", "/wire/empty"]
+WIRE_FETCH = """async paths => {
+ const bodies = {};
+ for (const path of paths) {
+ const response = await fetch(path, {method: path.endsWith('/head') ? 'HEAD' : 'GET'});
+ const body = await response.arrayBuffer();
+ if (path.includes('charset') || path.endsWith('/binary')) bodies[path] = Array.from(new Uint8Array(body));
+ }
+ return bodies;
+}"""
+
+
+@dataclass
+class _Gate:
+ started: Event = field(default_factory=Event)
+ release: Event = field(default_factory=Event)
+ finished: Event = field(default_factory=Event)
+
+
+class _Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def log_message(self, format: str, *args: Any) -> None:
+ pass
+
+ def do_GET(self) -> None:
+ path = urlsplit(self.path).path
+ if path == "/api/fail":
+ self.connection.shutdown(SHUT_RDWR)
+ self.connection.close()
+ return
+ if path == "/api/pending":
+ if not getattr(self.server, "release_pending").wait(10):
+ getattr(self.server, "errors").append("Timed out waiting to release /api/pending")
+ return
+ status, body = 200, b'{"ok":true}'
+ content_type: str | None = "application/json"
+ if path in ("/", "/second", "/fresh", "/other"):
+ content_type, body = "text/html", HTML
+ elif path == "/blank":
+ content_type, body = "text/html", BLANK
+ elif path == "/style.css":
+ content_type, body = "text/css", b"body { color: black; }"
+ elif path == "/script.js":
+ content_type, body = "application/javascript", SCRIPT
+ elif path == "/image.png":
+ content_type, body = "image/png", PNG
+ elif path == "/api/status":
+ status, content_type, body = 503, "text/plain", b"Try later"
+ elif path == "/api/large":
+ content_type, body = "text/plain", LARGE
+ elif path == "/redirect":
+ status, body = 302, b""
+ elif path in {"/wire/chunked", "/wire/gzip"}:
+ content_type, body = "text/plain", compress(LARGE) if path.endswith("gzip") else LARGE
+ elif path == "/wire/binary":
+ content_type, body = "application/octet-stream", BINARY
+ elif path == "/wire/charset":
+ content_type, body = "text/plain; charset=iso-8859-15", CHARSET_BODY
+ elif path == "/wire/charset-spaced":
+ content_type, body = 'text/plain; CHARSET = "iso-8859-15"', CHARSET_BODY
+ elif path == "/wire/raw-charset":
+ content_type, body = 'application/octet-stream; CHARSET = "iso-8859-15"', CHARSET_BODY
+ elif path == "/wire/head":
+ content_type = "application/octet-stream"
+ elif path == "/wire/empty":
+ status, content_type, body = 204, "application/octet-stream", b""
+ elif path in MIME_RESPONSES:
+ content_type, body = MIME_RESPONSES[path]
+ self.send_response(status)
+ if content_type is not None:
+ self.send_header("Content-Type", content_type)
+ self.send_header("Transfer-Encoding", "chunked") if path == "/wire/chunked" else self.send_header(
+ "Content-Length", str(len(body))
+ )
+ self.send_header("X-Recorded", "response")
+ if path == "/redirect":
+ self.send_header("Location", "/redirected")
+ if path == "/wire/gzip":
+ self.send_header("Content-Encoding", "gzip")
+ self.end_headers()
+ gate = getattr(self.server, "gates").get(path)
+ try:
+ if gate is not None:
+ self.wfile.write(body[:1])
+ self.wfile.flush()
+ gate.started.set()
+ if not gate.release.wait(10):
+ getattr(self.server, "errors").append(f"Timed out waiting to release {path}")
+ return
+ body = body[1:]
+ if path == "/wire/chunked":
+ for start in range(0, len(body), 65536):
+ chunk = body[start : start + 65536]
+ self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n")
+ self.wfile.write(b"0\r\n\r\n")
+ elif self.command != "HEAD":
+ self.wfile.write(body)
+ except (BrokenPipeError, ConnectionResetError):
+ pass
+ finally:
+ if gate is not None:
+ gate.finished.set()
+
+ def do_HEAD(self) -> None:
+ self.do_GET()
+
+ def do_POST(self) -> None:
+ body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
+ self.send_response(201)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("X-Recorded", "response")
+ self.end_headers()
+ self.wfile.write(body)
+
+
[email protected](scope="module")
+def release_pending() -> Event:
+ return Event()
+
+
[email protected](scope="module")
+def network_gates() -> dict[str, _Gate]:
+ return {}
+
+
[email protected](scope="module")
+def network_url(release_pending: Event, network_gates: dict[str, _Gate]) -> Generator[str, None, None]:
+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+ setattr(server, "release_pending", release_pending)
+ setattr(server, "gates", network_gates)
+ errors: list[str] = []
+ setattr(server, "errors", errors)
+ thread = Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_port}"
+ finally:
+ release_pending.set()
+ for gate in network_gates.values():
+ gate.release.set()
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=5)
+ assert not thread.is_alive()
+ assert not errors
+
+
+def _options(**kwargs: Any) -> dict[str, Any]:
+ return {
+ "executable_path": getenv("SCRAPLING_EXECUTABLE_PATH"),
+ "google_search": False,
+ "record_requests": True,
+ "retries": 1,
+ **kwargs,
+ }
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_disabled_by_default(session_type: Any) -> None:
+ session = session_type()
+ context = Mock()
+ session._initialize_context(session._config, context)
+ assert not session.network.enabled
+ context.on.assert_not_called()
+ assert session.network.search() == [] and session.network.last_id == 0
+ assert not session.network._contexts
+ session.close()
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_disabled_by_default(session_type: Any) -> None:
+ session = session_type()
+ context = Mock()
+ await session._initialize_context(session._config, context)
+ assert not session.network.enabled
+ context.on.assert_not_called()
+ assert session.network.search() == [] and session.network.last_id == 0
+ assert not session.network._contexts
+ await session.close()
+
+
+def _wait_saved(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:
+ for _ in range(250):
+ records = network.search(url_pattern=pattern, after_id=after_id)
+ if len(records) >= count:
+ return records
+ page.wait_for_timeout(20)
+ raise AssertionError(f"Expected {count} saved responses for {pattern!r}, got {len(records)}")
+
+
+async def _wait_saved_async(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:
+ for _ in range(250):
+ records = network.search(url_pattern=pattern, after_id=after_id)
+ if len(records) >= count:
+ return records
+ await asyncio.sleep(0.02)
+ raise AssertionError(f"Expected {count} saved responses for {pattern!r}, got {len(records)}")
+
+
+def _check_initial(network: Any, response: Any) -> Any:
+ records = network.search()
+ assert network.enabled
+ assert {record.resource_type for record in records} >= {"document", "script", "stylesheet", "image", "fetch", "xhr"}
+ assert len({record.id for record in records}) == len(records)
+ initial = network.search(url_pattern=r"/api/initial$", resource_type="fetch")
+ assert len(initial) == 1
+ assert initial[0].status == 200
+ assert initial[0].request_body is None
+ assert initial[0].response.json() == {"ok": True}
+ assert "body_note" not in initial[0].response.meta and not hasattr(initial[0], "request")
+ assert {urlsplit(item.url).path for item in response.captured_xhr} == {"/api/initial", "/api/xhr"}
+ assert all(item.json() == {"ok": True} for item in response.captured_xhr)
+ image = network.search(url_pattern=r"/image\.png$")[0]
+ assert image.response.body == b"" and image.response.meta.get("body_note") == "Non-text body; not saved."
+ for path, body in ((r"/script\.js$", SCRIPT), (r"/style\.css$", b"body { color: black; }")):
+ asset = network.search(url_pattern=path)[0]
+ assert asset.response.body == body and "body_note" not in asset.response.meta
+ filtered = network.search(include_static=False)
+ assert filtered and all(record.resource_type in ("document", "xhr", "fetch") for record in filtered)
+ return initial[0]
+
+
+def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:
+ records = network.search(after_id=after_id)
+ assert records and all(record.id > after_id for record in records)
+ post = network.search(method="POST", status=201)
+ assert len(post) == 1
+ assert loads(post[0].request_body) == {"sent": True}
+ duplicate = network.search(url_pattern=r"/api/repeated$")
+ assert len(duplicate) == 2 and duplicate[0].id != duplicate[1].id
+ assert len(network.search(url_pattern=r"/api/repeated$", limit=1)) == 1
+ status = network.search(status=503)
+ assert len(status) == 1
+ assert not network.search(url_pattern=r"/api/fail$")
+ redirect = network.search(url_pattern=r"/redirect$")
+ assert len(redirect) == 1 and redirect[0].status == 302
+ redirected = network.search(url_pattern=r"/redirected$", status=200)
+ assert len(redirected) == 1 and redirected[0].id != redirect[0].id
+ assert redirect[0].response_headers["location"] == "/redirected"
+ return post[0], status[0], network.search(url_pattern=r"/api/large$")[0]
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
+ with session_type(
+ **_options(
+ capture_xhr=r"/api/",
+ page_action=lambda page: page.locator("body[data-initial=ready]").wait_for(state="attached"),
+ )
+ ) as session:
+ response = session.fetch(network_url + "/")
+ page = session.page_pool.pages[0].page
+ _wait_saved(page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6)
+ initial = _check_initial(session.network, response)
+ assert initial.response.json() == {"ok": True}
+ page = session.page_pool.pages[0].page
+ after_id = session.network.last_id
+ with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):
+ page.evaluate(LATER)
+ _wait_saved(page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7)
+ post, status, large = _check_later(session.network, after_id)
+ assert post.response.request_headers["x-sent"] == "request"
+ assert post.response.headers["x-recorded"] == "response"
+ assert post.response.json() == {"sent": True}
+ assert status.response.body == b"Try later"
+ assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""
+ release_pending.clear()
+ try:
+ with page.expect_request_finished(lambda request: request.url.endswith("/api/pending")):
+ with page.expect_request("**/api/pending"):
+ page.evaluate(PENDING)
+ assert not session.network.search(url_pattern=r"/api/pending$")
+ release_pending.set()
+ pending = _wait_saved(page, session.network, r"/api/pending$")[0]
+ assert pending.status == 200 and pending.response.json() == {"ok": True}
+ finally:
+ release_pending.set()
+ other = session.context.new_page()
+ try:
+ other.goto(network_url + "/other")
+ other.locator("body[data-initial=ready]").wait_for(state="attached")
+ _wait_saved(other, session.network, r"/other$")
+ finally:
+ other.close()
+ assert session.network.search(url_pattern=r"/other$", resource_type="document")
+ session.fetch(network_url + "/second")
+ assert session.page_pool.pages[0].page is page
+ session.close_pages()
+ assert page.is_closed() and session.network.get(post.id).url == post.url
+ fresh_id = session.network.last_id
+ session.fetch(network_url + "/fresh")
+ _wait_saved(session.page_pool.pages[0].page, session.network, r"/(?:fresh|api/initial)$", 2, fresh_id)
+ assert session.page_pool.pages[0].page is not page
+ assert session.network.search(url_pattern=r"/fresh$", resource_type="document")
+ unread = session.network.search(url_pattern=r"/api/initial$")[-1]
+ page = session.page_pool.pages[0].page
+ with page.expect_request_finished(lambda request: request.url.endswith("/api/closing")):
+ page.evaluate("fetch('/api/closing', {method: 'POST', body: 'closing'}).then(response => response.text())")
+ _wait_saved(page, session.network, r"/api/closing$")
+ last_id = session.network.last_id
+ assert session.network.last_id == last_id
+ assert session.network.get(post.id).status == 201
+ assert session.network.get(post.id).method == "POST"
+ closing = session.network.search(url_pattern=r"/api/closing$")[0]
+ assert closing.response.body == b"closing" and "body_note" not in closing.response.meta
+ assert unread.response.json() == {"ok": True}
+ assert initial.response.json() == {"ok": True}
+ assert post.response.json() == {"sent": True}
+ assert post.response.request_headers["x-sent"] == "request"
+ assert post.response.headers["x-recorded"] == "response"
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
+ async def initial_page(page: Any) -> None:
+ await page.locator("body[data-initial=ready]").wait_for(state="attached")
+
+ async with session_type(**_options(capture_xhr=r"/api/", page_action=initial_page)) as session:
+ response = await session.fetch(network_url + "/")
+ page = session.page_pool.pages[0].page
+ await _wait_saved_async(
+ page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6
+ )
+ initial = _check_initial(session.network, response)
+ assert initial.response.json() == {"ok": True}
+ page = session.page_pool.pages[0].page
+ after_id = session.network.last_id
+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):
+ await page.evaluate(LATER)
+ await _wait_saved_async(
+ page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7
+ )
+ post, status, large = _check_later(session.network, after_id)
+ assert post.response.request_headers["x-sent"] == "request"
+ assert post.response.headers["x-recorded"] == "response"
+ assert post.response.json() == {"sent": True}
+ assert status.response.body == b"Try later"
+ assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""
+ release_pending.clear()
+ try:
+ async with page.expect_request("**/api/pending"):
+ await page.evaluate(PENDING)
+ assert not session.network.search(url_pattern=r"/api/pending$")
+ release_pending.set()
+ pending = (await _wait_saved_async(page, session.network, r"/api/pending$"))[0]
+ assert pending.status == 200 and pending.response.json() == {"ok": True}
+ assert not any("/api/pending" in record.url for record in response.captured_xhr)
+ finally:
+ release_pending.set()
+ other = await session.context.new_page()
+ try:
+ await other.goto(network_url + "/other")
+ await other.locator("body[data-initial=ready]").wait_for(state="attached")
+ await _wait_saved_async(other, session.network, r"/other$")
+ finally:
+ await other.close()
+ assert session.network.search(url_pattern=r"/other$", resource_type="document")
+ await session.fetch(network_url + "/second")
+ assert session.page_pool.pages[0].page is page
+ await session.close_pages()
+ assert page.is_closed() and session.network.get(post.id).url == post.url
+ fresh_id = session.network.last_id
+ await session.fetch(network_url + "/fresh")
+ await _wait_saved_async(
+ session.page_pool.pages[0].page, session.network, r"/(?:fresh|api/initial)$", 2, fresh_id
+ )
+ assert session.page_pool.pages[0].page is not page
+ assert session.network.search(url_pattern=r"/fresh$", resource_type="document")
+ unread = session.network.search(url_pattern=r"/api/initial$")[-1]
+ page = session.page_pool.pages[0].page
+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/closing")):
+ await page.evaluate(
+ "fetch('/api/closing', {method: 'POST', body: 'closing'}).then(response => response.text())"
+ )
+ await _wait_saved_async(page, session.network, r"/api/closing$")
+ last_id = session.network.last_id
+ assert session.network.last_id == last_id
+ assert session.network.get(post.id).status == 201
+ assert session.network.get(post.id).method == "POST"
+ closing = session.network.search(url_pattern=r"/api/closing$")[0]
+ assert closing.response.body == b"closing" and "body_note" not in closing.response.meta
+ assert unread.response.json() == {"ok": True}
+ assert initial.response.json() == {"ok": True}
+ assert post.response.json() == {"sent": True}
+ assert post.response.request_headers["x-sent"] == "request"
+ assert post.response.headers["x-recorded"] == "response"
+
+
+def _check_eviction(network: Any) -> int:
+ records = network.search()
+ assert len(records) == 2 and network.dropped_count > 0
+ assert network.get(1) is None
+ last_id = network.last_id
+ network.clear()
+ assert network.search() == [] and network.last_id == last_id
+ return last_id
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:
+ with session_type(**_options(max_recorded_requests=2)) as session:
+ session.fetch(network_url + "/")
+ page = session.page_pool.pages[0].page
+ page.locator("body[data-initial=ready]").wait_for(state="attached")
+ for _ in range(250):
+ if session.network.last_id >= 6:
+ break
+ page.wait_for_timeout(20)
+ assert session.network.last_id >= 6
+ last_id = _check_eviction(session.network)
+ with page.expect_request_finished(lambda request: request.url.endswith("/api/post")):
+ page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")
+ _wait_saved(page, session.network, r"/api/post$")
+ records = session.network.search()
+ assert len(records) == 1 and records[0].id > last_id
+ assert records[0].request_body == records[0].response.body == b"fresh"
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:
+ async with session_type(**_options(max_recorded_requests=2)) as session:
+ await session.fetch(network_url + "/")
+ page = session.page_pool.pages[0].page
+ await page.locator("body[data-initial=ready]").wait_for(state="attached")
+ for _ in range(250):
+ if session.network.last_id >= 6:
+ break
+ await asyncio.sleep(0.02)
+ assert session.network.last_id >= 6
+ last_id = _check_eviction(session.network)
+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/post")):
+ await page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")
+ await _wait_saved_async(page, session.network, r"/api/post$")
+ records = session.network.search()
+ assert len(records) == 1 and records[0].id > last_id
+ assert records[0].request_body == records[0].response.body == b"fresh"
+
+
+PROXY_HTML = '<html><body><script>fetch("/api").then(response => response.json()).then(value => document.body.dataset.api = JSON.stringify(value));</script></body></html>'
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> None:
+ def setup(page: Any) -> None:
+ page.route(
+ "**/*",
+ lambda route: route.fulfill(
+ content_type="application/json" if route.request.url.endswith("/api") else "text/html",
+ body='{"temporary":true}' if route.request.url.endswith("/api") else PROXY_HTML,
+ ),
+ )
+
+ def initial(page: Any) -> None:
+ page.locator("body[data-api]").wait_for(state="attached")
+ _wait_saved(page, session.network, r"/api$")
+
+ with session_type(
+ **_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]), page_setup=setup, page_action=initial)
+ ) as session:
+ session.fetch("https://network.test/")
+ assert not session.page_pool.pages
+ record = session.network.search(url_pattern=r"/api$")[0]
+ assert record.status == 200 and record.resource_type == "fetch"
+ assert record.response.json() == {"temporary": True}
+ assert session.network.get(record.id).url == "https://network.test/api"
+ assert record.response.json() == {"temporary": True}
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_browser_temporary_proxy_context(session_type: Any) -> None:
+ async def setup(page: Any) -> None:
+ await page.route(
+ "**/*",
+ lambda route: route.fulfill(
+ content_type="application/json" if route.request.url.endswith("/api") else "text/html",
+ body='{"temporary":true}' if route.request.url.endswith("/api") else PROXY_HTML,
+ ),
+ )
+
+ async def initial(page: Any) -> None:
+ await page.locator("body[data-api]").wait_for(state="attached")
+ await _wait_saved_async(page, session.network, r"/api$")
+
+ async with session_type(
+ **_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]), page_setup=setup, page_action=initial)
+ ) as session:
+ await session.fetch("https://network.test/")
+ assert not session.page_pool.pages
+ record = session.network.search(url_pattern=r"/api$")[0]
+ assert record.status == 200 and record.resource_type == "fetch"
+ assert record.response.json() == {"temporary": True}
+ assert session.network.get(record.id).url == "https://network.test/api"
+ assert record.response.json() == {"temporary": True}
+
+
+def _check_saved_override(network: Any, network_url: str) -> None:
+ records = network.search(url_pattern=r"/api/overridden$", method="POST", status=201)
+ assert len(records) == 1
+ record = records[0]
+ assert record.url == network_url + "/api/overridden"
+ assert record.request_body == record.response.body == b"overridden"
+ assert record.request_headers["x-overridden"] == "yes"
+ assert record.response.request_headers["x-overridden"] == "yes"
+ assert not network.search(url_pattern=r"/api/original$")
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_browser_keeps_route_overrides_after_close(
+ session_type: Any, network_url: str, monkeypatch: Any
+) -> None:
+ logger = Mock()
+ monkeypatch.setattr("scrapling.engines.toolbelt.custom.log", logger)
+ with session_type(**_options()) as session:
+ session.fetch(network_url + "/")
+ page = session.page_pool.pages[0].page
+ page.route(
+ "**/api/original",
+ lambda route: route.continue_(
+ url=network_url + "/api/overridden",
+ method="POST",
+ post_data="overridden",
+ headers={**route.request.headers, "x-overridden": "yes"},
+ ),
+ )
+ with page.expect_request_finished(lambda request: request.url.endswith("/api/overridden")):
+ page.evaluate("fetch('/api/original').then(response => response.text())")
+ _wait_saved(page, session.network, r"/api/overridden$")
+ _check_saved_override(session.network, network_url)
+ logger.info.assert_called_once()
+ assert f"<GET {network_url}/>" in logger.info.call_args.args[0]
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_browser_keeps_route_overrides_after_close(
+ session_type: Any, network_url: str, monkeypatch: Any
+) -> None:
+ logger = Mock()
+ monkeypatch.setattr("scrapling.engines.toolbelt.custom.log", logger)
+ async with session_type(**_options()) as session:
+ await session.fetch(network_url + "/")
+ page = session.page_pool.pages[0].page
+ await page.route(
+ "**/api/original",
+ lambda route: route.continue_(
+ url=network_url + "/api/overridden",
+ method="POST",
+ post_data="overridden",
+ headers={**route.request.headers, "x-overridden": "yes"},
+ ),
+ )
+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/overridden")):
+ await page.evaluate("fetch('/api/original').then(response => response.text())")
+ await _wait_saved_async(page, session.network, r"/api/overridden$")
+ _check_saved_override(session.network, network_url)
+ logger.info.assert_called_once()
+ assert f"<GET {network_url}/>" in logger.info.call_args.args[0]
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
[email protected]("target", ["page", "context", "session"])
+def test_sync_network_close_during_unfinished_response(
+ session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]
+) -> None:
+ path = f"/api/held/{session_type.__name__}/{target}"
+ gate = network_gates[path] = _Gate()
+ with session_type(**_options()) as session:
+ session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ saved = _wait_saved(page, session.network, r"/blank$")[0]
+ try:
+ page.evaluate("fetch('/api/fail').catch(() => null)")
+ with page.expect_response(network_url + path):
+ page.evaluate("path => { fetch(path).then(response => response.text()).catch(() => null); }", path)
+ assert gate.started.wait(5), "The server did not start the held response"
+ assert not gate.release.is_set() and not gate.finished.is_set()
+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")
+ started = monotonic()
+ {"page": session.close_pages, "context": session.context.close, "session": session.close}[target]()
+ assert monotonic() - started < 5
+ assert not gate.release.is_set() and not gate.finished.is_set()
+ assert page.is_closed()
+ assert session.network.get(saved.id).response.body == BLANK
+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")
+ assert bool(session.network._contexts) == (target == "page")
+ finally:
+ gate.release.set()
+ assert gate.finished.wait(5), "The server did not finish the released response"
+ assert not session.network._contexts
+ assert session.network.search() == [saved]
+ assert saved.response.body == BLANK
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("target", ["page", "context", "session"])
+async def test_async_network_close_during_unfinished_response(
+ session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]
+) -> None:
+ path = f"/api/held/{session_type.__name__}/{target}"
+ gate = network_gates[path] = _Gate()
+ async with session_type(**_options()) as session:
+ await session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ saved = (await _wait_saved_async(page, session.network, r"/blank$"))[0]
+ try:
+ await page.evaluate("fetch('/api/fail').catch(() => null)")
+ async with page.expect_response(network_url + path):
+ await page.evaluate(
+ "path => { fetch(path).then(response => response.text()).catch(() => null); }", path
+ )
+ assert await asyncio.to_thread(gate.started.wait, 5), "The server did not start the held response"
+ assert not gate.release.is_set() and not gate.finished.is_set()
+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")
+ close = {"page": session.close_pages, "context": session.context.close, "session": session.close}[target]
+ await asyncio.wait_for(close(), timeout=5)
+ assert not gate.release.is_set() and not gate.finished.is_set()
+ assert page.is_closed()
+ assert session.network.get(saved.id).response.body == BLANK
+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")
+ assert bool(session.network._contexts) == (target == "page")
+ finally:
+ gate.release.set()
+ assert await asyncio.to_thread(gate.finished.wait, 5), "The server did not finish the released response"
+ assert not session.network._contexts
+ assert session.network.search() == [saved]
+ assert saved.response.body == BLANK
+
+
+def _check_burst(network: Any, cursor: int, path: str, expected_paths: list[str]) -> int:
+ newer = network.search(after_id=cursor, limit=1)
+ assert len(newer) == 1 and urlsplit(newer[0].url).path == path
+ assert newer[0].id == cursor + 1
+ records = network.search(limit=2)
+ records += network.search(after_id=records[-1].id, limit=2)
+ assert [urlsplit(record.url).path for record in records] == expected_paths[-3:]
+ assert all(record.response.json() == {"ok": True} for record in records)
+ assert network.dropped_count == max(0, len(expected_paths) - 3)
+ return newer[0].id
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_burst_completion_order_and_pagination(
+ session_type: Any, network_url: str, network_gates: dict[str, _Gate]
+) -> None:
+ paths = [f"/api/burst/{session_type.__name__}/{index}" for index in range(6)]
+ gates = {path: _Gate() for path in paths}
+ network_gates.update(gates)
+ with session_type(**_options(max_recorded_requests=3)) as session:
+ session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ _wait_saved(page, session.network, r"/blank$")
+ session.network.clear()
+ cursor = session.network.last_id
+ completed: list[str] = []
+ try:
+ page.evaluate(
+ "paths => { for (const path of paths) fetch(path).then(response => response.text()).catch(() => null); }",
+ paths,
+ )
+ assert all(gate.started.wait(5) for gate in gates.values()), "The request burst did not reach the server"
+ assert session.network.search() == []
+ for index in (5, 2, 4, 1, 3, 0):
+ path = paths[index]
+ gates[path].release.set()
+ _wait_saved(page, session.network, path + "$", after_id=cursor)
+ completed.append(path)
+ cursor = _check_burst(session.network, cursor, path, completed)
+ assert session.network.get(cursor - 3) is None
+ finally:
+ for gate in gates.values():
+ gate.release.set()
+ assert all(gate.finished.wait(5) for gate in gates.values())
+ assert not session.network._contexts
+ records = session.network.search()
+ assert [urlsplit(record.url).path for record in records] == completed[-3:]
+ assert all(record.response.json() == {"ok": True} for record in records)
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_burst_completion_order_and_pagination(
+ session_type: Any, network_url: str, network_gates: dict[str, _Gate]
+) -> None:
+ paths = [f"/api/burst/{session_type.__name__}/{index}" for index in range(6)]
+ gates = {path: _Gate() for path in paths}
+ network_gates.update(gates)
+ async with session_type(**_options(max_recorded_requests=3)) as session:
+ await session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ await _wait_saved_async(page, session.network, r"/blank$")
+ session.network.clear()
+ cursor = session.network.last_id
+ completed: list[str] = []
+ try:
+ await page.evaluate(
+ "paths => { for (const path of paths) fetch(path).then(response => response.text()).catch(() => null); }",
+ paths,
+ )
+ for gate in gates.values():
+ assert await asyncio.to_thread(gate.started.wait, 5), "The request burst did not reach the server"
+ assert session.network.search() == []
+ for index in (5, 2, 4, 1, 3, 0):
+ path = paths[index]
+ gates[path].release.set()
+ await _wait_saved_async(page, session.network, path + "$", after_id=cursor)
+ completed.append(path)
+ cursor = _check_burst(session.network, cursor, path, completed)
+ assert session.network.get(cursor - 3) is None
+ finally:
+ for gate in gates.values():
+ gate.release.set()
+ for gate in gates.values():
+ assert await asyncio.to_thread(gate.finished.wait, 5)
+ assert not session.network._contexts
+ records = session.network.search()
+ assert [urlsplit(record.url).path for record in records] == completed[-3:]
+ assert all(record.response.json() == {"ok": True} for record in records)
+
+
+def _check_wire_bodies(
+ network: Any, wire: dict[str, list[int]], native_bodies: dict[str, bytes], native_texts: dict[str, str]
+) -> None:
+ records = {urlsplit(record.url).path: record for record in network.search(url_pattern=r"/wire/")}
+ assert set(records) == set(WIRE_PATHS)
+ for path in ("/wire/chunked", "/wire/gzip"):
+ assert records[path].response.meta.get("body_note") == "too large; not saved."
+ assert records[path].response.body == b""
+ assert "content-length" not in records["/wire/chunked"].response_headers
+ assert records["/wire/chunked"].response_headers["transfer-encoding"] == "chunked"
+ assert int(records["/wire/gzip"].response_headers["content-length"]) < 1024 * 1024
+ assert records["/wire/gzip"].response_headers["content-encoding"] == "gzip"
+ assert bytes(wire["/wire/binary"]) == native_bodies["/wire/binary"] == BINARY
+ for path in ("/wire/binary", "/wire/raw-charset"):
+ assert records[path].response.body == b""
+ assert records[path].response.meta.get("body_note") == "Non-text body; not saved."
+ details = _request_details(records[path], "response_body")
+ assert details.data is None and details.note == "Non-text body; not saved."
+ for path in CHARSET_PATHS:
+ assert bytes(wire[path]) == CHARSET_BODY
+ assert records[path].response_headers["content-type"].lower().endswith(("iso-8859-15", '"iso-8859-15"'))
+ for path, text in native_texts.items():
+ assert records[path].response.body == native_bodies[path]
+ assert "body_note" not in records[path].response.meta
+ assert records[path].response.encoding == "utf-8"
+ assert native_bodies[path].decode("utf-8") == text
+ assert records[path].response.get_all_text().strip() == text
+ details = _request_details(records[path], "response_body")
+ assert details.data == text and details.request_id == records[path].id
+ assert details.part == "response_body" and details.note is None
+ assert native_texts["/wire/charset"] == "caf\u00e9 \u20ac"
+ assert native_bodies["/wire/charset"] == "caf\u00e9 \u20ac".encode("utf-8")
+ assert native_bodies["/wire/raw-charset"] == CHARSET_BODY
+ assert records["/wire/head"].method == "HEAD" and records["/wire/head"].status == 200
+ assert records["/wire/empty"].status == 204
+ for path in ("/wire/head", "/wire/empty"):
+ assert records[path].response.body == b"" and "body_note" not in records[path].response.meta
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:
+ with session_type(**_options()) as session:
+ session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ native: dict[str, Any] = {}
+ page.on("response", lambda response: native.update({urlsplit(response.url).path: response}))
+ wire = page.evaluate(WIRE_FETCH, WIRE_PATHS)
+ _wait_saved(page, session.network, r"/wire/", len(WIRE_PATHS))
+ native_bodies = {path: native[path].body() for path in wire}
+ native_texts = {path: native[path].text() for path in CHARSET_PATHS[:2]}
+ with pytest.raises(UnicodeDecodeError):
+ native["/wire/raw-charset"].text()
+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)
+ assert not session.network._contexts
+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:
+ async with session_type(**_options()) as session:
+ await session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ native: dict[str, Any] = {}
+ page.on("response", lambda response: native.update({urlsplit(response.url).path: response}))
+ wire = await page.evaluate(WIRE_FETCH, WIRE_PATHS)
+ await _wait_saved_async(page, session.network, r"/wire/", len(WIRE_PATHS))
+ native_bodies = {path: await native[path].body() for path in wire}
+ native_texts = {path: await native[path].text() for path in CHARSET_PATHS[:2]}
+ with pytest.raises(UnicodeDecodeError):
+ await native["/wire/raw-charset"].text()
+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)
+ assert not session.network._contexts
+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)
+
+
+def _check_mime_bodies(network: Any, wire: dict[str, list[int]], reads: list[str]) -> None:
+ records = {urlsplit(record.url).path: record for record in network.search(url_pattern=r"/mime/")}
+ assert set(records) == set(wire) == set(MIME_RESPONSES)
+ assert sorted(reads) == sorted(TEXT_RESPONSES)
+ for path, (content_type, body) in MIME_RESPONSES.items():
+ record = records[path]
+ assert bytes(wire[path]) == body
+ assert record.status == 200 and record.method == "GET" and record.resource_type == "fetch"
+ assert record.response_headers.get("content-type") == content_type
+ assert record.response_headers["x-recorded"] == "response"
+ assert int(record.response_headers["content-length"]) == len(body)
+ if path in TEXT_RESPONSES:
+ assert record.response.body == body and "body_note" not in record.response.meta
+ else:
+ assert record.response.body == b""
+ assert record.response.meta.get("body_note") == "Non-text body; not saved."
+ for path in ("/mime/json", "/mime/vendor-json"):
+ assert records[path].response.json() == {"saved": True}
+
+
+ ("session_type", "response_type"), [(DynamicSession, PlaywrightResponse), (StealthySession, PatchrightResponse)]
+)
+def test_sync_network_reads_only_text_bodies(
+ session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ reads: list[str] = []
+ native_body = response_type.body
+
+ def read_body(response: Any) -> bytes:
+ path = urlsplit(response.url).path
+ if path in MIME_RESPONSES:
+ reads.append(path)
+ return native_body(response)
+
+ monkeypatch.setattr(response_type, "body", read_body)
+ with session_type(**_options()) as session:
+ session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ wire = page.evaluate(MIME_FETCH, list(MIME_RESPONSES))
+ _wait_saved(page, session.network, r"/mime/", len(MIME_RESPONSES))
+ _check_mime_bodies(session.network, wire, reads)
+ assert not session.network._contexts
+ _check_mime_bodies(session.network, wire, reads)
+
+
+ ("session_type", "response_type"),
+ [(AsyncDynamicSession, AsyncPlaywrightResponse), (AsyncStealthySession, AsyncPatchrightResponse)],
+)
+async def test_async_network_reads_only_text_bodies(
+ session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ reads: list[str] = []
+ native_body = response_type.body
+
+ async def read_body(response: Any) -> bytes:
+ path = urlsplit(response.url).path
+ if path in MIME_RESPONSES:
+ reads.append(path)
+ return await native_body(response)
+
+ monkeypatch.setattr(response_type, "body", read_body)
+ async with session_type(**_options()) as session:
+ await session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ wire = await page.evaluate(MIME_FETCH, list(MIME_RESPONSES))
+ await _wait_saved_async(page, session.network, r"/mime/", len(MIME_RESPONSES))
+ _check_mime_bodies(session.network, wire, reads)
+ assert not session.network._contexts
+ _check_mime_bodies(session.network, wire, reads)
diff --git a/tests/fetchers/test_network_cleanup.py b/tests/fetchers/test_network_cleanup.py
new file mode 100644
index 0000000..dc67a75
--- /dev/null
+++ b/tests/fetchers/test_network_cleanup.py
@@ -0,0 +1,83 @@
+import asyncio
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from scrapling.engines._browsers._base import AsyncSession, SyncSession
+from tests.fetchers.test_network_history import request
+
+
+def _session(asynchronous=False):
+ session = AsyncSession(record_requests=True) if asynchronous else SyncSession(record_requests=True)
+ context, browser, playwright = Mock(), Mock(), Mock()
+ if asynchronous:
+ context.close, browser.close, playwright.stop = AsyncMock(), AsyncMock(), AsyncMock()
+ session.context, session.browser, session.playwright = context, browser, playwright
+ session._is_alive = True
+ session.network._attach(context)
+ capture = dict(call.args for call in context.on.call_args_list)["requestfinished"]
+ return session, context, browser, playwright, capture
+
+
+def _closed(session, context, browser, playwright):
+ assert not session._is_alive and not session.network._contexts
+ assert session.context is session.browser is session.playwright is None
+ context.close.assert_called_once_with()
+ browser.close.assert_called_once_with()
+ playwright.stop.assert_called_once_with()
+ assert {call.args[0] for call in context.remove_listener.call_args_list} == {"requestfinished", "close"}
+
+
+def test_sync_session_close_during_capture_does_not_restore_late_response():
+ session, context, browser, playwright, capture = _session()
+ capture(request(body=b"saved"))
+ saved = session.network.get(1)
+
+ def close_before_body_returns():
+ session.close()
+ return b"late"
+
+ native = request()
+ native.existing_response.body.side_effect = close_before_body_returns
+ capture(native)
+ session.close()
+ _closed(session, context, browser, playwright)
+ assert session.network.search() == [saved] and saved.response.body == b"saved"
+ assert session.network.last_id == 1
+
+
[email protected]("read_error", [False, True])
+async def test_async_session_close_does_not_wait_for_running_capture(read_error):
+ session, context, browser, playwright, capture = _session(asynchronous=True)
+ await capture(request(body=b"saved", asynchronous=True))
+ saved = session.network.get(1)
+ reading, release = asyncio.Event(), asyncio.Event()
+
+ async def held_body():
+ reading.set()
+ await release.wait()
+ if read_error:
+ raise RuntimeError("Browser closed while reading")
+ return b"late"
+
+ native = request(asynchronous=True)
+ native.existing_response.body.side_effect = held_body
+ task = asyncio.create_task(capture(native))
+ try:
+ await asyncio.wait_for(reading.wait(), timeout=2)
+ await asyncio.wait_for(session.close(), timeout=2)
+ assert not task.done() and not release.is_set()
+ _closed(session, context, browser, playwright)
+ context.close.assert_awaited_once_with()
+ browser.close.assert_awaited_once_with()
+ playwright.stop.assert_awaited_once_with()
+ release.set()
+ await asyncio.wait_for(task, timeout=2)
+ await session.close()
+ assert session.network.search() == [saved] and saved.response.body == b"saved"
+ assert session.network.last_id == 1
+ finally:
+ release.set()
+ await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=2)
+ await session.close()
diff --git a/tests/fetchers/test_network_config.py b/tests/fetchers/test_network_config.py
new file mode 100644
index 0000000..585afe2
--- /dev/null
+++ b/tests/fetchers/test_network_config.py
@@ -0,0 +1,32 @@
+import pytest
+
+from scrapling.core._types import Any
+from scrapling.engines._browsers._validators import PlaywrightConfig, StealthConfig, validate
+from scrapling.fetchers import DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession
+
+
[email protected]("model", [PlaywrightConfig, StealthConfig])
+def test_network_config_defaults(model: type[PlaywrightConfig]) -> None:
+ config = validate({}, model)
+ assert config.record_requests is False
+ assert config.max_recorded_requests == 1000
+
+
[email protected]("model", [PlaywrightConfig, StealthConfig])
+ "values", [{"record_requests": "yes"}, {"max_recorded_requests": 0}, {"max_recorded_requests": -1}]
+)
+def test_network_config_invalid(model: type[PlaywrightConfig], values: dict[str, Any]) -> None:
+ with pytest.raises(TypeError):
+ validate(values, model)
+
+
[email protected]("session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession])
[email protected]("enabled", [False, True])
+def test_sessions_configure_one_network_history(session_type: Any, enabled: bool) -> None:
+ session = session_type(record_requests=enabled, max_recorded_requests=7)
+ assert session.network.enabled is enabled
+ assert session._config.max_recorded_requests == 7
+ assert session.network.search() == []
+ assert session.network.get(1) is None
+ assert session.network.last_id == 0
diff --git a/tests/fetchers/test_network_history.py b/tests/fetchers/test_network_history.py
new file mode 100644
index 0000000..1743b0b
--- /dev/null
+++ b/tests/fetchers/test_network_history.py
@@ -0,0 +1,559 @@
+import asyncio
+import gc
+from inspect import isawaitable
+from weakref import ref
+from unittest.mock import AsyncMock, Mock, PropertyMock
+
+import pytest
+
+from scrapling.core._types import Any, Optional
+from scrapling.engines._browsers import _network
+from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest
+from scrapling.engines.toolbelt.custom import Response
+
+
+class Context:
+ def __init__(self):
+ self.listeners: dict[str, list[Any]] = {}
+
+ def on(self, event, listener):
+ self.listeners.setdefault(event, []).append(listener)
+
+ def remove_listener(self, event, listener):
+ self.listeners[event].remove(listener)
+
+ def emit(self, event, value=None):
+ tasks = []
+ for listener in self.listeners.get(event, []).copy():
+ result = listener(value)
+ if isawaitable(result):
+ tasks.append(asyncio.ensure_future(result))
+ return tasks
+
+
+def request(
+ url="https://example.test/api",
+ method="GET",
+ resource_type="fetch",
+ status: Optional[int] = 200,
+ body=b'{"ok":true}',
+ asynchronous=False,
+):
+ make = AsyncMock if asynchronous else Mock
+ native = Mock(url=url, method=method, resource_type=resource_type)
+ native.headers = {"cookie": "session=1"}
+ native.all_headers = make(return_value=native.headers.copy())
+ native.post_data_buffer = None
+ native.existing_response = None
+ if status is not None:
+ response = Mock(url=url, status=status, status_text="OK", request=native)
+ response.headers = {"content-type": "application/json", "content-length": str(len(body))}
+ response.all_headers = make(return_value=response.headers.copy())
+ response.body = make(return_value=body)
+ native.existing_response = response
+ return native
+
+
+def recorder(max_requests=1000, asynchronous=False):
+ network = NetworkRecorder(enabled=True, max_requests=max_requests, asynchronous=asynchronous)
+ context = Context()
+ network._attach(context)
+ return network, context
+
+
+async def finish(context, native):
+ await asyncio.gather(*context.emit("requestfinished", native))
+
+
[email protected]("asynchronous", [False, True])
+async def test_only_completed_responses_are_saved(asynchronous):
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(asynchronous=asynchronous)
+ assert set(context.listeners) == {"requestfinished", "close"}
+ context.emit("request", native)
+ context.emit("response", native.existing_response)
+ context.emit("requestfailed", request(status=None, asynchronous=asynchronous))
+ assert network.search() == [] and network.last_id == 0
+ native.all_headers.assert_not_called()
+ native.existing_response.body.assert_not_called()
+ await finish(context, request(status=None, asynchronous=asynchronous))
+ assert network.search() == []
+ await finish(context, native)
+ record = network.get(1)
+ assert isinstance(record, NetworkRequest) and not hasattr(record, "request")
+ assert not hasattr(record, "state") and not hasattr(record, "failure")
+ assert record.id == 1 and record.url == native.url and record.method == "GET"
+ assert record.resource_type == "fetch" and record.status == 200
+ assert record.response.json() == {"ok": True}
+ assert "body_note" not in record.response.meta
+
+
[email protected]("asynchronous", [False, True])
+async def test_saved_response_has_final_request_data_and_survives_close(asynchronous):
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(asynchronous=asynchronous)
+ original_url = native.url
+ context.emit("request", native)
+ native.url, native.method = "https://example.test/overridden", "POST"
+ native.headers = {"x-overridden": "yes", "content-type": "text/plain"}
+ native.post_data_buffer = b"overridden"
+ native.all_headers.return_value = native.headers.copy()
+ native.existing_response.url = native.url
+ await finish(context, native)
+ record = network.get(1)
+ assert record is not None
+ assert record.url == native.url and record.response.url == native.url and record.method == "POST"
+ assert record.request_body == b"overridden"
+ assert record.request_headers == record.response.request_headers == native.headers
+ assert record.response_headers == record.response.headers == native.existing_response.headers
+ assert network.search(url_pattern=original_url) == [] and network.search(method="POST") == [record]
+ native.headers["x-overridden"] = "changed"
+ native.existing_response.headers["content-type"] = "changed"
+ assert record.request_headers["x-overridden"] == "yes"
+ assert record.response_headers["content-type"] == "application/json"
+ native.all_headers.assert_called_once_with()
+ native.existing_response.all_headers.assert_called_once_with()
+ native.existing_response.body.assert_called_once_with()
+ context.emit("close")
+ network._detach()
+ native.all_headers.side_effect = RuntimeError("Browser closed")
+ native.existing_response.all_headers.side_effect = RuntimeError("Browser closed")
+ native.existing_response.body.side_effect = RuntimeError("Browser closed")
+ assert record.response.json() == {"ok": True}
+ assert network.get(1) is record and network.search() == [record]
+ native.existing_response.body.assert_called_once_with()
+
+
[email protected]("asynchronous", [False, True])
[email protected]("outcome", ["saved", "oversized", "unreadable", "invalid_charset", "non_text"])
+async def test_recording_skips_fetch_log_without_muting_normal_responses(monkeypatch, asynchronous, outcome):
+ logger = Mock()
+ monkeypatch.setattr("scrapling.engines.toolbelt.custom.log", logger)
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(asynchronous=asynchronous)
+ if outcome == "oversized":
+ monkeypatch.setattr(_network, "MAX_BODY_BYTES", 4)
+ elif outcome == "unreadable":
+ native.existing_response.body.side_effect = RuntimeError("Body missing")
+ elif outcome == "invalid_charset":
+ native.existing_response.headers["content-type"] = "application/json; charset=bogus"
+ elif outcome == "non_text":
+ native.existing_response.headers["content-type"] = "application/octet-stream"
+ await finish(context, native)
+ record = network.get(1)
+ assert record is not None
+ assert (
+ record.response.meta
+ == {
+ "saved": {},
+ "oversized": {"body_note": "too large; not saved."},
+ "unreadable": {"body_note": "Body missing"},
+ "invalid_charset": {},
+ "non_text": {"body_note": "Non-text body; not saved."},
+ }[outcome]
+ )
+ logger.info.assert_not_called()
+ response = Response(
+ url="https://example.test/page",
+ content=b"page",
+ status=200,
+ reason="OK",
+ cookies={},
+ headers={},
+ request_headers={},
+ )
+ assert response.body == b"page"
+ logger.info.assert_called_once_with("Fetched (200) <GET https://example.test/page> (referer: None)")
+
+
[email protected]("asynchronous", [False, True])
+async def test_oversized_response_keeps_headers_without_reading_body(monkeypatch, asynchronous):
+ monkeypatch.setattr(_network, "MAX_BODY_BYTES", 4)
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(body=b"large", asynchronous=asynchronous)
+ await finish(context, native)
+ record = network.get(1)
+ assert record is not None
+ assert record.status == 200 and record.response.meta == {"body_note": "too large; not saved."}
+ assert record.response.body == b"" and record.response.headers["content-length"] == "5"
+ native.existing_response.body.assert_not_called()
+
+
[email protected]("asynchronous", [False, True])
[email protected]("message", ["Body was removed", ""])
+async def test_unreadable_response_keeps_headers_and_reports_unavailable(asynchronous, message):
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(asynchronous=asynchronous)
+ native.existing_response.body.side_effect = RuntimeError(message)
+ await finish(context, native)
+ record = network.get(1)
+ assert record is not None
+ assert record.status == 200
+ assert record.response.meta == {"body_note": message or "could not be saved."}
+ assert record.response.body == b"" and record.response.headers == native.existing_response.headers
+
+
[email protected]("asynchronous", [False, True])
+async def test_non_text_records_keep_metadata_without_using_body_budget(monkeypatch, asynchronous):
+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 8)
+ network, context = recorder(asynchronous=asynchronous)
+ await finish(context, request(body=b"1234", asynchronous=asynchronous))
+ for request_id, content_type in enumerate(("application/octet-stream", "application/x-custom", None), start=2):
+ native = request(
+ url=f"https://example.test/binary/{request_id}",
+ method="POST",
+ resource_type="image",
+ status=206,
+ body=b"\x00\xffbinary",
+ asynchronous=asynchronous,
+ )
+ native.post_data_buffer = b"submitted"
+ response = native.existing_response
+ response.headers = {"x-saved": "yes"}
+ if content_type is not None:
+ response.headers["content-type"] = content_type
+ response.all_headers.return_value = response.headers.copy()
+ response.body.side_effect = AssertionError("Non-text body must not be read")
+ await finish(context, native)
+ record = network.get(request_id)
+ assert record is not None
+ assert record.url == native.url and record.method == "POST" and record.status == 206
+ assert record.resource_type == "image" and record.request_body == b"submitted"
+ assert record.request_headers == native.headers and record.response_headers == response.headers
+ assert record.response.body == b"" and record.response.meta == {"body_note": "Non-text body; not saved."}
+ response.body.assert_not_called()
+ await finish(context, request(body=b"5678", asynchronous=asynchronous))
+ assert [record.id for record in network.search()] == [1, 2, 3, 4, 5]
+ assert [record.response.body for record in network.search()] == [b"1234", b"", b"", b"", b"5678"]
+ assert network.dropped_count == 0
+ await finish(context, request(body=b"9", asynchronous=asynchronous))
+ assert [record.id for record in network.search()] == [2, 3, 4, 5, 6]
+ assert network.get(1) is None and network.dropped_count == 1
+
+
[email protected]("asynchronous", [False, True])
+async def test_total_body_budget_evicts_oldest_records_and_clear_releases_budget(monkeypatch, asynchronous):
+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 7)
+ network, context = recorder(asynchronous=asynchronous)
+ for body in (b"one", b"two", b"three"):
+ await finish(context, request(body=body, asynchronous=asynchronous))
+ assert [record.id for record in network.search()] == [3]
+ record = network.get(3)
+ assert record is not None
+ assert network.dropped_count == 2 and record.response.body == b"three"
+ network.clear()
+ assert network.dropped_count == 0 and network.last_id == 3 and not network.search()
+ for _ in range(2):
+ await finish(context, request(body=b"new", asynchronous=asynchronous))
+ assert [record.id for record in network.search()] == [4, 5] and network.dropped_count == 0
+
+
[email protected]("asynchronous", [False, True])
+async def test_count_eviction_releases_saved_body_budget(monkeypatch, asynchronous):
+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 5)
+ network, context = recorder(max_requests=1, asynchronous=asynchronous)
+ for _ in range(3):
+ await finish(context, request(body=b"new", asynchronous=asynchronous))
+ assert [record.id for record in network.search()] == [3]
+ record = network.get(3)
+ assert record is not None
+ assert record.response.body == b"new" and network.dropped_count == 2
+ network.clear()
+ await finish(context, request(body=b"new", asynchronous=asynchronous))
+ assert [record.id for record in network.search()] == [4] and network.dropped_count == 0
+
+
[email protected]("change", ["clear", "detach", "reattach"])
[email protected]("started", [False, True])
+async def test_late_callback_cannot_publish_after_clear_or_detach(change, started):
+ network, context = recorder(asynchronous=True)
+ reading, release = asyncio.Event(), asyncio.Event()
+
+ async def body():
+ reading.set()
+ await release.wait()
+ return b"old"
+
+ native = request(asynchronous=True)
+ native.existing_response.body.side_effect = body
+ tasks = context.emit("requestfinished", native)
+ try:
+ if started:
+ await asyncio.wait_for(reading.wait(), timeout=2)
+ if change == "clear":
+ network.clear()
+ else:
+ network._detach(context)
+ if change == "reattach":
+ network._attach(context)
+ release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=2)
+ assert network.search() == [] and network.last_id == 0
+ if change != "detach":
+ await finish(context, request(body=b"new", asynchronous=True))
+ record = network.get(1)
+ assert record is not None
+ assert record.response.body == b"new"
+ finally:
+ release.set()
+ await asyncio.gather(*tasks)
+ network._detach()
+
+
+async def test_records_are_published_in_save_order_without_waiting_for_other_bodies():
+ network, context = recorder(asynchronous=True)
+ reading, release = asyncio.Event(), asyncio.Event()
+
+ async def body():
+ reading.set()
+ await release.wait()
+ return b"first"
+
+ first = request(asynchronous=True)
+ first.existing_response.body.side_effect = body
+ tasks = context.emit("requestfinished", first)
+ try:
+ await asyncio.wait_for(reading.wait(), timeout=2)
+ assert network.search() == [] and network.get(1) is None
+ await finish(context, request(body=b"second", asynchronous=True))
+ record = network.get(1)
+ assert record is not None
+ assert record.response.body == b"second"
+ release.set()
+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=2)
+ assert [record.response.body for record in network.search()] == [b"second", b"first"]
+ finally:
+ release.set()
+ await asyncio.gather(*tasks)
+
+
+def test_context_lifecycle_keeps_history_and_removes_only_owned_listeners():
+ network, first = recorder()
+ second, external = Context(), Mock()
+ first.on("requestfinished", external)
+ network._attach(first)
+ network._attach(second)
+ assert len(first.listeners["requestfinished"]) == 2 and len(first.listeners["close"]) == 1
+ first.emit("requestfinished", request(status=201))
+ second.emit("requestfinished", request())
+ network._detach(first)
+ first.emit("requestfinished", request())
+ second.emit("requestfinished", request())
+ assert network.last_id == 3
+ assert first.listeners["requestfinished"] == [external] and not first.listeners["close"]
+ second.emit("close")
+ assert not any(second.listeners.values()) and not network._contexts
+ record = network.get(1)
+ assert record is not None
+ assert record.status == 201
+ network._attach(first)
+ first.emit("requestfinished", request())
+ network._detach()
+ network._detach()
+ assert network.enabled and len(network.search()) == 4
+ assert first.listeners["requestfinished"] == [external] and not first.listeners["close"]
+ external.assert_called()
+
+
+def test_search_filters_include_completed_http_errors():
+ network, context = recorder()
+ for native in (
+ request(url="https://example.test/page", resource_type="document"),
+ request(url="https://example.test/api", method="POST", status=503),
+ request(url="https://example.test/missing", resource_type="document", status=404),
+ request(url="https://example.test/api", resource_type="xhr"),
+ ):
+ context.emit("requestfinished", native)
+ assert [record.id for record in network.search(include_static=False)] == [2, 3, 4]
+ assert [record.id for record in network.search(after_id=2, limit=1)] == [3]
+ assert [record.id for record in network.search(url_pattern="/api$", method="post", status=503)] == [2]
+ assert [record.id for record in network.search(resource_type="xhr")] == [4]
+ assert network.get(100) is None and network.search(url_pattern="absent") == []
+
+
[email protected]("options", [{"limit": 0}, {"after_id": -1}, {"url_pattern": "["}])
+def test_invalid_filters_raise_value_error(options):
+ with pytest.raises(ValueError):
+ NetworkRecorder().search(**options)
+
+
+def test_disabled_recorder_never_attaches():
+ network, context = NetworkRecorder(), Mock()
+ network._attach(context)
+ network._detach(context)
+ network._detach()
+ assert not context.mock_calls and not network.search() and network.last_id == 0
+ with pytest.raises(ValueError, match="greater than zero"):
+ NetworkRecorder(max_requests=0)
+
+
+def test_partial_listener_setup_failure_does_not_leave_capture_attached():
+ network, context = NetworkRecorder(enabled=True), Mock()
+ context.on.side_effect = [None, RuntimeError("Context closed")]
+ network._attach(context)
+ assert not network._contexts
+ assert context.remove_listener.call_count == 2
+
+
[email protected]("asynchronous", [False, True])
[email protected]("outcome", ["finished", "body_error", "headers_error", "conversion_error", "unfinished"])
+async def test_native_handles_are_not_kept_in_history(asynchronous, outcome):
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(asynchronous=asynchronous)
+ native_ref, response_ref = ref(native), ref(native.existing_response)
+ if outcome == "body_error":
+ native.existing_response.body.side_effect = RuntimeError("Body missing")
+ elif outcome == "headers_error":
+ native.existing_response.all_headers.side_effect = RuntimeError("Headers missing")
+ elif outcome == "conversion_error":
+ type(native.existing_response).status_text = PropertyMock(side_effect=RuntimeError("Response missing"))
+ if outcome != "unfinished":
+ await finish(context, native)
+ else:
+ context.emit("request", native)
+ if outcome in {"unfinished", "conversion_error"}:
+ assert network.search() == []
+ else:
+ record = network.get(1)
+ assert record is not None
+ assert record.status == 200
+ assert record.response.meta.get("body_note") == ("Body missing" if outcome == "body_error" else None)
+ del native
+ gc.collect()
+ assert native_ref() is None and response_ref() is None
+
+
[email protected]("change", ["clear", "detach", "reattach"])
+def test_sync_conversion_cannot_restore_history_after_reentrant_reset(change):
+ network, context = recorder()
+ native = request(body=b"old")
+
+ def body():
+ assert network.search() == [] and network.get(1) is None
+ if change == "clear":
+ network.clear()
+ else:
+ network._detach(context)
+ if change == "reattach":
+ network._attach(context)
+ if change != "detach":
+ context.emit("requestfinished", request(body=b"new"))
+ return b"old"
+
+ native.existing_response.body.side_effect = body
+ context.emit("requestfinished", native)
+ expected = [] if change == "detach" else [b"new"]
+ assert [record.response.body for record in network.search()] == expected
+ assert network.last_id == len(expected)
+
+
+async def test_cancelled_conversion_publishes_nothing_and_later_requests_still_save():
+ network, context = recorder(asynchronous=True)
+ reading, release = asyncio.Event(), asyncio.Event()
+
+ async def body():
+ reading.set()
+ await release.wait()
+ return b"cancelled"
+
+ native = request(asynchronous=True)
+ native.existing_response.body.side_effect = body
+ tasks = context.emit("requestfinished", native)
+ try:
+ await asyncio.wait_for(reading.wait(), timeout=2)
+ tasks[0].cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await tasks[0]
+ assert network.search() == [] and network.last_id == 0
+ await finish(context, request(body=b"saved", asynchronous=True))
+ record = network.get(1)
+ assert record is not None
+ assert record.response.body == b"saved"
+ finally:
+ release.set()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ network._detach()
+
+
[email protected]("asynchronous", [False, True])
[email protected]("budget", ["count", "body", "both"])
+async def test_exact_storage_boundaries_keep_records_until_exceeded(monkeypatch, asynchronous, budget):
+ monkeypatch.setattr(_network, "MAX_BODY_BYTES", 4)
+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 100 if budget == "count" else 8)
+ network, context = recorder(max_requests=3 if budget == "body" else 2, asynchronous=asynchronous)
+ for body in (b"1111", b"2222"):
+ await finish(context, request(body=body, asynchronous=asynchronous))
+ assert [record.response.body for record in network.search()] == [b"1111", b"2222"]
+ assert network.dropped_count == 0
+ await finish(context, request(body=b"3", asynchronous=asynchronous))
+ assert [record.response.body for record in network.search()] == [b"2222", b"3"]
+ assert network.get(1) is None and network.dropped_count == 1
+
+
[email protected]("asynchronous", [False, True])
[email protected]("attribute", ["resource_type", "post_data_buffer"])
+async def test_unreadable_request_metadata_is_omitted_and_native_handles_are_released(asynchronous, attribute):
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(asynchronous=asynchronous)
+ native_ref, response_ref = ref(native), ref(native.existing_response)
+ setattr(type(native), attribute, PropertyMock(side_effect=RuntimeError("Request metadata missing")))
+ await finish(context, native)
+ assert network.search() == []
+ del native
+ gc.collect()
+ assert native_ref() is None and response_ref() is None
+ await finish(context, request(body=b"next", asynchronous=asynchronous))
+ assert [record.response.body for record in network.search()] == [b"next"]
+
+
[email protected]("asynchronous", [False, True])
+async def test_detach_ignores_removed_listener_errors_and_rejects_stale_callbacks(asynchronous):
+ network, context = recorder(asynchronous=asynchronous)
+ context.remove_listener = Mock(side_effect=RuntimeError("Context already closed"))
+ network._detach(context)
+ await finish(context, request(asynchronous=asynchronous))
+ assert network.search() == [] and network.last_id == 0
+ context.remove_listener.assert_any_call("requestfinished", context.listeners["requestfinished"][0])
+ context.remove_listener.assert_any_call("close", context.listeners["close"][0])
+
+
[email protected]("asynchronous", [False, True])
+ "body,expected,encoding",
+ [
+ ("café €".encode(), "café €", "utf-8"),
+ (b"caf\xe9 \xa4", "café €", "iso-8859-15"),
+ (b"\xc3\xa9", "é", "utf-8"),
+ ],
+)
+async def test_saved_native_bytes_and_text_encoding_remain_valid_after_detach(asynchronous, body, expected, encoding):
+ network, context = recorder(asynchronous=asynchronous)
+ native = request(body=body, asynchronous=asynchronous)
+ content_type = "text/plain; charset=iso-8859-15"
+ native.existing_response.headers["content-type"] = content_type
+ native.existing_response.all_headers.return_value = native.existing_response.headers.copy()
+ await finish(context, native)
+ network._detach()
+ record = network.get(1)
+ assert record is not None
+ assert record.response.body == body
+ assert record.response_headers["content-type"] == content_type
+ assert record.response.encoding == encoding
+ assert record.response.get_all_text().strip() == expected
+ assert "body_note" not in record.response.meta
diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py
index b422ead..152d525 100644
--- a/tests/fetchers/test_response_handling.py
+++ b/tests/fetchers/test_response_handling.py
@@ -18,6 +18,8 @@ def make_sync_playwright_response():
response.headers = {"content-type": LATIN9_CONTENT_TYPE}
response.all_headers = Mock(return_value={"content-type": LATIN9_CONTENT_TYPE})
response.body = Mock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))
+ response.request.method = "GET"
+ response.request.headers = {}
response.request.all_headers = Mock(return_value={})
response.request.redirected_from = None
return response
@@ -39,6 +41,8 @@ def make_async_playwright_response():
response.headers = {"content-type": LATIN9_CONTENT_TYPE}
response.all_headers = AsyncMock(return_value={"content-type": LATIN9_CONTENT_TYPE})
response.body = AsyncMock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))
+ response.request.method = "GET"
+ response.request.headers = {}
response.request.all_headers = AsyncMock(return_value={})
response.request.redirected_from = None
return response
@@ -70,10 +74,7 @@ class TestResponseFactory:
mock_curl_response.request.method = "GET"
mock_curl_response.history = []
- response = ResponseFactory.from_http_request(
- mock_curl_response,
- {"adaptive": False}
- )
+ response = ResponseFactory.from_http_request(mock_curl_response, {"adaptive": False})
assert response.status == 200
assert response.url == "https://example.com"
@@ -96,9 +97,15 @@ class TestResponseFactory:
assert "Bürger" in response.html_content
assert "BÃ" not in response.html_content
- def test_playwright_raw_response_keeps_header_encoding(self):
+ @pytest.mark.parametrize(
+ "content_type",
+ [LATIN9_CONTENT_TYPE, 'text/html; CHARSET = "iso-8859-15"', "text/html; charset = 'iso-8859-15'"],
+ )
+ def test_playwright_raw_response_keeps_header_encoding(self, content_type):
"""Raw response bytes should still use the charset from Content-Type."""
mock_response = make_sync_playwright_response()
+ mock_response.headers = {"content-type": content_type}
+ mock_response.all_headers.return_value = mock_response.headers
response = ResponseFactory.from_playwright_response(
None,
@@ -130,9 +137,15 @@ class TestResponseFactory:
assert "BÃ" not in response.html_content
@pytest.mark.asyncio
- async def test_async_playwright_raw_response_keeps_header_encoding(self):
+ @pytest.mark.parametrize(
+ "content_type",
+ [LATIN9_CONTENT_TYPE, 'text/html; CHARSET = "iso-8859-15"', "text/html; charset = 'iso-8859-15'"],
+ )
+ async def test_async_playwright_raw_response_keeps_header_encoding(self, content_type):
"""Raw response bytes should still use the charset from Content-Type."""
mock_response = make_async_playwright_response()
+ mock_response.headers = {"content-type": content_type}
+ mock_response.all_headers.return_value = mock_response.headers
response = await ResponseFactory.from_async_playwright_response(
None,
@@ -145,30 +158,317 @@ class TestResponseFactory:
assert response.encoding == "iso-8859-15"
assert "Bürger" in response.html_content
- def test_response_history_processing(self):
- """Test processing response history"""
- # Mock responses with redirects
- mock_final = Mock()
- mock_final.status = 200
- mock_final.status_text = "OK"
- mock_final.all_headers = Mock(return_value={})
-
- mock_redirect = Mock()
- mock_redirect.url = "https://example.com/redirect"
- mock_redirect.response = Mock(return_value=mock_final)
- mock_redirect.all_headers = Mock(return_value={})
- mock_redirect.redirected_from = None
-
- mock_first = Mock()
- mock_first.request.redirected_from = mock_redirect
-
- # Process history
- history = ResponseFactory._process_response_history(
- mock_first,
- {}
- )
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("asynchronous", [False, True])
+ async def test_response_history_processing(self, asynchronous):
+ make = make_async_playwright_response if asynchronous else make_sync_playwright_response
+ make_method = AsyncMock if asynchronous else Mock
+ first, middle, final = make(), make(), make()
+ for native, path, status in ((first, "start", 301), (middle, "middle", 302), (final, "final", 200)):
+ native.url = native.request.url = f"https://example.com/{path}"
+ native.status = status
+ native.status_text = ""
+ native.request.response = make_method(return_value=native)
+ native.request.all_headers.return_value = {"x-redirect": path}
+ final.request.redirected_from = middle.request
+ middle.request.redirected_from = first.request
+ if asynchronous:
+ response = await ResponseFactory.from_async_playwright_response(None, final, None, {})
+ else:
+ response = ResponseFactory.from_playwright_response(None, final, None, {})
+ assert response.status == 200 and response.url == "https://example.com/final"
+ assert [(item.url, item.status, item.reason) for item in response.history] == [
+ ("https://example.com/start", 301, "Moved Permanently"),
+ ("https://example.com/middle", 302, "Found"),
+ ]
+ assert [item.request_headers for item in response.history] == [
+ {"x-redirect": "start"},
+ {"x-redirect": "middle"},
+ ]
+ assert all(item.body == b"" and item.encoding == "iso-8859-15" for item in response.history)
+
+
[email protected]("asynchronous", [False, True])
+class TestRecordedResponseFactory:
+ @staticmethod
+ async def convert(native, asynchronous, **kwargs):
+ if asynchronous:
+ return await ResponseFactory.from_async_playwright_response(
+ None, native, None, {}, collect_history=False, **kwargs
+ )
+ return ResponseFactory.from_playwright_response(None, native, None, {}, collect_history=False, **kwargs)
+
+ @staticmethod
+ def native(asynchronous, body=b"data", headers=None):
+ native = make_async_playwright_response() if asynchronous else make_sync_playwright_response()
+ native.headers = {"content-type": "text/plain"} if headers is None else headers
+ native.all_headers.return_value = native.headers
+ native.body.return_value = body
+ return native
+
+ async def test_saved_response_keeps_bytes_and_full_headers(self, asynchronous):
+ native = self.native(asynchronous, b'\x00{"data": "saved"}\x00')
+ native.all_headers.return_value = {**native.headers, "set-cookie": "session=value"}
+ native.request.all_headers.return_value = {"cookie": "session=value"}
+ metadata = {"proxy": "http://proxy"}
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100, meta=metadata)
- assert len(history) >= 0 # Should process redirects
+ assert isinstance(response, Response)
+ assert response.body == b'\x00{"data": "saved"}\x00'
+ assert response.headers["set-cookie"] == "session=value"
+ assert response.request_headers["cookie"] == "session=value"
+ assert response.meta == {"proxy": "http://proxy"}
+ assert metadata == {"proxy": "http://proxy"}
+
+ @pytest.mark.parametrize("method,status", [("HEAD", 200), ("GET", 204), ("GET", 205), ("GET", 304)])
+ @pytest.mark.parametrize("declared_length", [None, "1000000"])
+ async def test_absent_bodies_are_not_read(self, asynchronous, method, status, declared_length):
+ native = self.native(asynchronous, headers={})
+ native.request.method, native.status = method, status
+ if declared_length is not None:
+ native.headers.update({"content-type": "application/octet-stream", "content-length": declared_length})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == b""
+ assert response.meta == {}
+ native.body.assert_not_called()
+
+ async def test_content_length_skips_large_body(self, asynchronous):
+ native = self.native(asynchronous, headers={"content-type": "text/plain", "content-length": "101"})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == b""
+ assert response.meta == {"body_note": "too large; not saved."}
+ native.body.assert_not_called()
+
+ @pytest.mark.parametrize("content_type", ["application/octet-stream", "image/png", "application/x-custom", None])
+ async def test_non_text_bodies_are_not_read_but_metadata_is_saved(self, asynchronous, content_type):
+ headers = {} if content_type is None else {"content-type": content_type}
+ native = self.native(asynchronous, b"\x00\xffbinary", headers)
+ native.status, native.status_text, native.request.method = 206, "Partial Content", "POST"
+ native.all_headers.return_value = {**headers, "x-saved": "response"}
+ native.request.all_headers.return_value = {"x-saved": "request"}
+ native.body.side_effect = AssertionError("Non-text body must not be read")
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == b""
+ assert response.meta == {"body_note": "Non-text body; not saved."}
+ assert response.url == native.url and response.status == 206 and response.reason == "Partial Content"
+ assert response.headers == {**headers, "x-saved": "response"}
+ assert response.request_headers == {"x-saved": "request"}
+ native.body.assert_not_called()
+
+ @pytest.mark.parametrize(
+ "content_type,body",
+ [
+ (' Text/Plain ; charset="utf-8"', "café".encode()),
+ ("application/json", b'{"saved": true}'),
+ ("application/xml", b"<saved>true</saved>"),
+ ("application/javascript", b"const saved = true;"),
+ ("application/x-javascript", b"var saved = true;"),
+ ("application/graphql", b"{ saved }"),
+ ("application/x-www-form-urlencoded", b"saved=true"),
+ ("application/problem+json", b'{"saved": true}'),
+ ("image/svg+xml", b"<svg></svg>"),
+ ],
+ )
+ async def test_text_content_types_keep_the_complete_body(self, asynchronous, content_type, body):
+ native = self.native(asynchronous, body, {"content-type": content_type})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=len(body))
+
+ assert response.body == body
+ assert response.headers == {"content-type": content_type}
+ assert response.meta == {}
+ native.body.assert_called_once_with()
+
+ @pytest.mark.parametrize("declared_length", [None, "1", "invalid"])
+ async def test_actual_size_rejects_whole_body(self, asynchronous, declared_length):
+ native = self.native(asynchronous, b"12345")
+ if declared_length is not None:
+ native.headers["content-length"] = declared_length
+
+ response = await self.convert(native, asynchronous, max_body_bytes=4)
+
+ assert response.body == b""
+ assert response.meta == {"body_note": "too large; not saved."}
+ native.body.assert_called_once()
+
+ @pytest.mark.parametrize("body,limit", [(b"1234", 4), (b"", 0)])
+ @pytest.mark.parametrize("declared", [False, True])
+ async def test_available_body_at_limit(self, asynchronous, body, limit, declared):
+ native = self.native(asynchronous, body)
+ if declared:
+ native.headers["content-length"] = str(len(body))
+
+ response = await self.convert(native, asynchronous, max_body_bytes=limit)
+
+ assert response.body == body
+ assert "body_note" not in response.meta
+
+ @pytest.mark.parametrize("message", ["Response body is unavailable", ""])
+ async def test_read_failure_is_not_an_empty_available_body(self, asynchronous, monkeypatch, message):
+ native = self.native(asynchronous)
+ native.body.side_effect = RuntimeError(message)
+ logger = Mock()
+ monkeypatch.setattr("scrapling.engines.toolbelt.convertor.log", logger)
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == b""
+ assert response.meta == {"body_note": message or "could not be saved."}
+ logger.error.assert_not_called()
+
+ async def test_header_failures_keep_partial_headers(self, asynchronous):
+ native = self.native(asynchronous)
+ native.all_headers.side_effect = RuntimeError("closed")
+ native.request.all_headers.side_effect = RuntimeError("closed")
+ native.request.headers = {"accept": "text/plain"}
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == b"data"
+ assert response.headers == {"content-type": "text/plain"}
+ assert response.request_headers == {"accept": "text/plain"}
+ assert response.meta == {"headers_partial": True, "request_headers_partial": True}
+
+ @pytest.mark.parametrize(
+ "body,content_type",
+ [(b"\xff\x00\xfe", "text/plain"), (b"\xff\x00\xfe", "text/plain; charset=bogus")],
+ )
+ async def test_parser_errors_do_not_drop_saved_bytes(self, asynchronous, body, content_type):
+ native = self.native(asynchronous, body, {"content-type": content_type})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == body
+ assert "body_note" not in response.meta
+ assert response.headers["content-type"] == content_type
+ assert isinstance(response.html_content, str)
+ if content_type.endswith("charset=bogus"):
+ assert response.encoding == "utf-8" and response.get_all_text() == ""
+
+ @pytest.mark.parametrize(
+ "body,expected,encoding",
+ [
+ ("café €".encode(), "café €", "utf-8"),
+ (b"caf\xe9 \xa4", "café €", "iso-8859-15"),
+ (b"\xc3\xa9", "é", "utf-8"),
+ ],
+ )
+ @pytest.mark.parametrize("recorded", [False, True])
+ async def test_native_bytes_and_selected_text_encoding(self, asynchronous, body, expected, encoding, recorded):
+ content_type = "text/plain; charset=iso-8859-15"
+ native = self.native(asynchronous, body, {"content-type": content_type})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100 if recorded else None)
+
+ assert response.body == body
+ assert response.headers["content-type"] == content_type
+ assert response.encoding == (encoding if recorded else "iso-8859-15")
+ assert response.get_all_text().strip() == (expected if recorded else body.decode("iso-8859-15"))
+ assert response.meta == {}
+ native.body.assert_called_once_with()
+
+ async def test_recorded_utf8_text_ignores_unknown_declared_charset(self, asynchronous):
+ body = '{"saved": "café €"}'.encode()
+ content_type = "application/json; charset=bogus"
+ native = self.native(asynchronous, body, {"content-type": content_type})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+
+ assert response.body == body and response.headers["content-type"] == content_type
+ assert response.encoding == "utf-8"
+ assert response.get_all_text().strip() == '{"saved": "café €"}'
+ assert response.json() == {"saved": "café €"}
+ assert response.meta == {}
+
+ async def test_unlimited_mode_does_not_add_recording_metadata(self, asynchronous):
+ native = self.native(asynchronous, b"data", {"content-length": "10000000"})
+
+ response = await self.convert(native, asynchronous)
+
+ assert response.body == b"data"
+ assert response.meta == {}
+ native.body.assert_called_once()
+
+ async def test_unlimited_mode_keeps_binary_bodies(self, asynchronous):
+ body = b"\x89PNG\r\n\x1a\n\x00binary"
+ native = self.native(asynchronous, body, {"content-type": "image/png"})
+
+ response = await self.convert(native, asynchronous)
+
+ assert response.body == body
+ assert response.headers == {"content-type": "image/png"}
+ assert response.meta == {}
+ native.body.assert_called_once_with()
+
+ @pytest.mark.parametrize("source", ["response", "request"])
+ async def test_unlimited_mode_keeps_header_errors(self, asynchronous, source):
+ native = self.native(asynchronous)
+ target = native if source == "response" else native.request
+ target.all_headers.side_effect = RuntimeError("closed")
+
+ with pytest.raises(RuntimeError, match="closed"):
+ await self.convert(native, asynchronous)
+
+ async def test_unlimited_mode_keeps_body_error_behavior(self, asynchronous, monkeypatch):
+ native = self.native(asynchronous)
+ native.body.side_effect = RuntimeError("closed")
+ logger = Mock()
+ monkeypatch.setattr("scrapling.engines.toolbelt.convertor.log", logger)
+
+ response = await self.convert(native, asynchronous)
+
+ assert response.body == b""
+ assert response.meta == {}
+ logger.error.assert_called_once()
+
+ async def test_unlimited_mode_keeps_encoding_errors(self, asynchronous):
+ native = self.native(asynchronous, headers={"content-type": "text/plain; charset=bogus"})
+
+ with pytest.raises(LookupError):
+ await self.convert(native, asynchronous)
+
+ @pytest.mark.parametrize("source", ["response", "request", "both"])
+ async def test_body_error_stays_unavailable_when_headers_also_fail(self, asynchronous, source):
+ native = self.native(asynchronous)
+ native.body.side_effect = RuntimeError("Body is gone")
+ if source in {"response", "both"}:
+ native.all_headers.side_effect = RuntimeError("Response headers are gone")
+ if source in {"request", "both"}:
+ native.request.all_headers.side_effect = RuntimeError("Request headers are gone")
+ response = await self.convert(native, asynchronous, max_body_bytes=100)
+ assert response.body == b""
+ assert response.meta["body_note"] == "Body is gone"
+ assert response.meta.get("headers_partial", False) == (source in {"response", "both"})
+ assert response.meta.get("request_headers_partial", False) == (source in {"request", "both"})
+ assert response.headers == native.headers and response.request_headers == native.request.headers
+
+ async def test_missing_response_raises_instead_of_returning_empty_data(self, asynchronous):
+ with pytest.raises(ValueError, match="Failed to get a response"):
+ await self.convert(None, asynchronous, max_body_bytes=100)
+
+ @pytest.mark.parametrize(
+ "content_type,body",
+ [("application/json", b'{"saved": true}'), ("application/octet-stream", b"\x00\xffbinary")],
+ )
+ async def test_recording_rules_do_not_change_captured_xhr(self, asynchronous, content_type, body):
+ native = self.native(asynchronous)
+ xhr = self.native(asynchronous, body, {"content-type": content_type})
+
+ response = await self.convert(native, asynchronous, max_body_bytes=4, xhr_captured=[xhr])
+
+ assert response.body == b"data"
+ assert response.captured_xhr[0].body == body
+ assert response.captured_xhr[0].headers == {"content-type": content_type}
+ assert response.captured_xhr[0].meta == {}
+ xhr.body.assert_called_once_with()
class TestErrorScenarios:
diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.py
index 35eeb85..8f36672 100644
--- a/tests/fetchers/test_utils.py
+++ b/tests/fetchers/test_utils.py
@@ -149,30 +149,46 @@ def test_unknown_status_code():
assert StatusText.get(1000) == "Unknown Status Code"
-# The private classmethod is name-mangled; resolve it once for the tests below.
-_extract_encoding = getattr(ResponseFactory, "_ResponseFactory__extract_browser_encoding")
-
-
def test_browser_encoding_unquoted_charset():
"""A charset declared without quotes is returned verbatim."""
- assert _extract_encoding("text/html; charset=utf-8") == "utf-8"
- assert _extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1"
- assert _extract_encoding("text/html;charset=windows-1252") == "windows-1252"
+ assert ResponseFactory._extract_encoding("text/html; charset=utf-8") == "utf-8"
+ assert ResponseFactory._extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1"
+ assert ResponseFactory._extract_encoding("text/html;charset=windows-1252") == "windows-1252"
def test_browser_encoding_quoted_charset():
"""A quoted charset value (RFC 7231 allows quoting) is unwrapped, not dropped."""
- assert _extract_encoding('text/html; charset="utf-8"') == "utf-8"
- assert _extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1"
- assert _extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS"
- assert _extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252"
+ assert ResponseFactory._extract_encoding('text/html; charset="utf-8"') == "utf-8"
+ assert ResponseFactory._extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1"
+ assert ResponseFactory._extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS"
+ assert ResponseFactory._extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252"
def test_browser_encoding_defaults_when_missing():
"""Fall back to the default when no charset is present or the header is empty."""
- assert _extract_encoding("text/html") == "utf-8"
- assert _extract_encoding("") == "utf-8"
- assert _extract_encoding(None) == "utf-8"
+ assert ResponseFactory._extract_encoding("text/html") == "utf-8"
+ assert ResponseFactory._extract_encoding("") == "utf-8"
+ assert ResponseFactory._extract_encoding(None) == "utf-8"
+
+
+ "header, expected",
+ [
+ ('text/plain; CHARSET = "ISO-8859-1"', "ISO-8859-1"),
+ ("text/plain; charset = utf-16", "utf-16"),
+ ('text/plain; charset=""', "utf-8"),
+ ('text/plain; note="charset=utf-16"', "utf-8"),
+ ("text/plain; x-charset=utf-16", "utf-8"),
+ ("text/plain; charset*=utf-8''UTF-16", "UTF-16"),
+ ],
+)
+def test_browser_encoding_header_parameters(header, expected):
+ assert ResponseFactory._extract_encoding(header) == expected
+
+
+def test_browser_encoding_custom_default():
+ assert ResponseFactory._extract_encoding(None, "ISO-8859-1") == "ISO-8859-1"
+ assert ResponseFactory._extract_encoding('text/plain; charset=""', "ISO-8859-1") == "ISO-8859-1"
class TestConstructProxyDict: