🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: Add coverage for the new parts
Karim shoair70b1916777ba
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex a61aa07..4cc5e3b 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -740,7 +740,7 @@ class TestModeSplitContract:assert result.structured_content is not Noneassert result.structured_content["session_type"] == "stealthy"session = _FakeStealthySession.instances[-1]- assert session.kwargs == {**options, "block_ads": True}+ assert session.kwargs == {**options, "block_ads": True, "record_requests": True}assert server._sessions["browser"].session is sessionclosed = await client.call_tool("close_session", {"session_id": "browser"})assert not closed.is_error@@ -980,7 +980,7 @@ class TestMCPServerAuthentication:monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)built = ScraplingMCPServer(auth_token=SHARED_KEY)._build_server("0.0.0.0", 8000)- assert len(built._tool_manager.list_tools()) == 14+ assert len(built._tool_manager.list_tools()) == 16def test_http_without_a_token_refuses_to_serve(self, monkeypatch):"""The streamable-http transport requires authentication unless the caller explicitly opts out"""@@ -1037,7 +1037,7 @@ class TestServerToolRegistration:@pytest.mark.asyncioasync def test_tools_are_listed_with_expected_schemas(self):- """All 14 tools are advertised, with plain content for screenshots, snapshots, actions, and JavaScript"""+ """All 16 tools are advertised, with plain content for screenshots, snapshots, actions, and JavaScript"""server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)async with Client(server) as client:assert client.instructions@@ -1067,7 +1067,7 @@ class TestServerToolRegistration:result = await client.call_tool(name, {})assert result.is_error- assert len(tools) == 14+ assert len(tools) == 16assert tools["browser_screenshot"].output_schema is Noneassert tools["browser_snapshot"].output_schema is Noneassert tools["browser_actions"].output_schema is None@@ -1175,7 +1175,7 @@ class TestServerToolRegistration:assert result.ttl_ms == 3_600_000 and result.cache_scope == "public"annotations = {tool.name: tool.annotations for tool in result.tools if tool.annotations is not None}- assert len(annotations) == 14+ assert len(annotations) == 16for name in ("make_request","bulk_get",diff --git a/tests/ai/test_browser_network.py b/tests/ai/test_browser_network.pynew file mode 100644index 0000000..712a771--- /dev/null+++ b/tests/ai/test_browser_network.py@@ -0,0 +1,1024 @@+from json import loads+from os import getenv+from asyncio import Event, create_task, sleep, wait_for+from dataclasses import replace+from types import SimpleNamespace+from urllib.parse import urlsplit+from unittest.mock import AsyncMock, Mock++import pytest+from mcp.client import Client+from mcp.types import CallToolResult, TextContent++from scrapling.core._types import Any+from scrapling.core._network_formatting import NetworkRequestModel, _request_details+from scrapling.core.ai import NetworkRequestsModel, ScraplingMCPServer, _SessionEntry+from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest+from scrapling.engines.toolbelt.custom import Response+++def _server() -> tuple[ScraplingMCPServer, Mock]:+ server = ScraplingMCPServer()+ saved = Response(+ url="https://network.test/api/items",+ content='{"name":"مرحبا"}'.encode(),+ status=201,+ reason="Created",+ cookies={},+ headers={"content-type": "application/json; charset=utf-8", "set-cookie": "test=2"},+ request_headers={"content-type": "application/json", "cookie": "test=1"},+ )+ record = NetworkRequest(+ id=4,+ url=saved.url,+ method="POST",+ resource_type="fetch",+ status=201,+ request_headers=saved.request_headers,+ request_body=b'{"sent":true}',+ response_headers=saved.headers,+ response=saved,+ )+ network = Mock(spec=["last_id", "dropped_count", "search", "get"], last_id=4, dropped_count=0)+ network.search.return_value = [record]+ network.get.side_effect = lambda request_id: record if request_id == 4 else None+ session = SimpleNamespace(_is_alive=True, network=network)+ server._sessions["browser"] = _SessionEntry(session, "stealthy")+ return server, network+++def _recording_server(max_requests: int = 3) -> tuple[ScraplingMCPServer, NetworkRecorder, Mock]:+ server, network, context = ScraplingMCPServer(), NetworkRecorder(True, max_requests, asynchronous=True), Mock()+ network._attach(context)+ session = SimpleNamespace(_is_alive=True, network=network)+ server._sessions["browser"] = _SessionEntry(session, "stealthy")+ return server, network, context+++def _native_request(path: str, resource_type: str = "fetch", status: int = 200) -> Mock:+ request = Mock(+ url=f"https://network.test{path}",+ method="GET",+ resource_type=resource_type,+ headers={},+ post_data_buffer=None,+ )+ request.all_headers = AsyncMock(return_value={})+ response = Mock(+ url=request.url,+ status=status,+ status_text="OK",+ request=request,+ headers={"content-type": "text/plain"},+ )+ response.all_headers = AsyncMock(return_value=response.headers.copy())+ response.body = AsyncMock(return_value=b"saved")+ request.existing_response = response+ return request+++def _listing(result: CallToolResult) -> dict[str, Any]:+ assert not result.is_error and isinstance(result.structured_content, dict)+ assert len(result.content) == 1 and isinstance(result.content[0], TextContent)+ assert loads(result.content[0].text) == result.structured_content+ assert set(result.structured_content) == {"requests", "next_cursor", "has_more", "dropped_count"}+ return result.structured_content+++def _detail(result: CallToolResult) -> dict[str, Any]:+ assert not result.is_error and isinstance(result.structured_content, dict)+ assert len(result.content) == 1 and isinstance(result.content[0], TextContent)+ assert loads(result.content[0].text) == result.structured_content+ assert set(result.structured_content) == {"request_id", "part", "data", "note"}+ return result.structured_content+++def _detail_info(+ request_id: int,+ part: str,+ data: Any = None,+ note: str | None = None,+) -> dict[str, Any]:+ return {+ "request_id": request_id,+ "part": part,+ "data": data,+ "note": note,+ }+++def _request_info(+ request_id: int, path: str, method: str = "GET", resource_type: str = "fetch", status: int = 200+) -> dict[str, Any]:+ return {+ "id": request_id,+ "url": f"https://network.test{path}",+ "method": method,+ "resource_type": resource_type,+ "status": status,+ }+++async def test_network_tools_schema_and_content() -> None:+ server, network = _server()+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ tools = {tool.name: tool for tool in (await client.list_tools()).tools}+ for name in ("browser_network_requests", "browser_network_request"):+ tool = tools[name]+ assert tool.annotations is not None and tool.annotations.read_only_hint is True+ assert tool.annotations.open_world_hint is False+ listing = tools["browser_network_requests"]+ assert listing.title == "List network requests"+ assert listing.input_schema["required"] == ["session_id"]+ properties = listing.input_schema["properties"]+ assert properties["include_static"]["default"] is False+ assert properties["limit"]["default"] == 50 and properties["limit"]["maximum"] == 200+ assert properties["after_id"]["minimum"] == 0+ schema = listing.output_schema+ assert schema is not None and schema["type"] == "object"+ output_properties = schema["properties"]+ assert {name: value["type"] for name, value in output_properties.items()} == {+ "requests": "array",+ "next_cursor": "integer",+ "has_more": "boolean",+ "dropped_count": "integer",+ }+ item_schema = schema["$defs"][output_properties["requests"]["items"]["$ref"].rsplit("/", 1)[-1]]+ assert item_schema["type"] == "object"+ assert {name: value["type"] for name, value in item_schema["properties"].items()} == {+ "id": "integer",+ "url": "string",+ "method": "string",+ "resource_type": "string",+ "status": "integer",+ }+ detail = tools["browser_network_request"]+ detail_schema = detail.output_schema+ assert detail_schema is not None and detail_schema["type"] == "object"+ output_properties = detail_schema["properties"]+ assert set(output_properties) == {"request_id", "part", "data", "note"}+ assert output_properties["request_id"]["type"] == "integer"+ assert output_properties["part"]["type"] == "string"+ assert set(output_properties["part"]["enum"]) == {+ "summary",+ "request_headers",+ "request_body",+ "response_headers",+ "response_body",+ }+ assert {option["type"] for option in output_properties["note"]["anyOf"]} == {"string", "null"}+ data_options = output_properties["data"]["anyOf"]+ assert len(data_options) == 4+ assert {option["type"] for option in data_options if "type" in option} == {"object", "string", "null"}+ header_schema = next(option for option in data_options if option.get("type") == "object")+ assert header_schema["additionalProperties"] == {"type": "string"}+ summary_ref = next(option["$ref"] for option in data_options if "$ref" in option)+ assert detail_schema["$defs"][summary_ref.rsplit("/", 1)[-1]] == item_schema+ assert detail.title == "Read network request"+ assert detail.input_schema["required"] == ["session_id", "request_id"]+ properties = detail.input_schema["properties"]+ assert properties["part"]["default"] == "summary"+ assert set(properties["part"]["enum"]) == {+ "summary",+ "request_headers",+ "request_body",+ "response_headers",+ "response_body",+ }+ assert set(properties) == {"session_id", "request_id", "part"}+ assert properties["request_id"]["minimum"] == 1+ result = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ assert _listing(result) == {+ "requests": [_request_info(4, "/api/items", "POST", status=201)],+ "next_cursor": 4,+ "has_more": False,+ "dropped_count": 0,+ }+ network.search.assert_called_once_with(url_pattern=None, include_static=False, after_id=0, limit=51)+++async def test_network_list_forwards_filters_and_reports_paging_eviction() -> None:+ server, network = _server()+ network.search.return_value.append(+ replace(+ network.get(4),+ id=8,+ url="https://network.test/next",+ method="GET",+ resource_type="document",+ )+ )+ network.last_id = 8+ network.dropped_count = 3+ result = await server.browser_network_requests(+ "browser", url_pattern="/api/", include_static=True, after_id=2, limit=1+ )+ network.search.assert_called_once_with(url_pattern="/api/", include_static=True, after_id=2, limit=2)+ assert isinstance(result, NetworkRequestsModel)+ assert result.model_dump() == {+ "requests": [_request_info(4, "/api/items", "POST", status=201)],+ "next_cursor": 4,+ "has_more": True,+ "dropped_count": 3,+ }+ network.search.return_value = []+ result = await server.browser_network_requests("browser", after_id=10)+ assert result.model_dump() == {"requests": [], "next_cursor": 10, "has_more": False, "dropped_count": 3}+++async def test_network_cursor_follows_save_order_and_survives_clear() -> None:+ server, network, context = _recording_server()+ callback = network._contexts[context]["requestfinished"]+ started, release = Event(), Event()+ slow, fast = _native_request("/api/slow"), _native_request("/api/fast")++ async def slow_body() -> bytes:+ started.set()+ await release.wait()+ return b"slow"++ slow.existing_response.body.side_effect = slow_body+ capture = create_task(callback(slow))+ try:+ await wait_for(started.wait(), timeout=2)+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ empty = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ assert _listing(empty) == {"requests": [], "next_cursor": 0, "has_more": False, "dropped_count": 0}+ await callback(fast)+ first = await client.call_tool("browser_network_requests", {"session_id": "browser", "limit": 1})+ assert _listing(first) == {+ "requests": [_request_info(1, "/api/fast")],+ "next_cursor": 1,+ "has_more": False,+ "dropped_count": 0,+ }+ release.set()+ await wait_for(capture, timeout=2)+ second = await client.call_tool(+ "browser_network_requests", {"session_id": "browser", "after_id": 1, "limit": 1}+ )+ assert _listing(second) == {+ "requests": [_request_info(2, "/api/slow")],+ "next_cursor": 2,+ "has_more": False,+ "dropped_count": 0,+ }+ network.clear()+ await callback(_native_request("/api/new"))+ after_clear = await client.call_tool("browser_network_requests", {"session_id": "browser", "after_id": 2})+ assert _listing(after_clear) == {+ "requests": [_request_info(3, "/api/new")],+ "next_cursor": 3,+ "has_more": False,+ "dropped_count": 0,+ }+ assert network.get(2) is None+ finally:+ release.set()+ await wait_for(capture, timeout=2)+ network._detach()+++async def test_network_real_filtering_eviction_and_cursor_paging() -> None:+ server, network, context = _recording_server()+ callback = network._contexts[context]["requestfinished"]+ try:+ for native in (+ _native_request("/page", "document"),+ _native_request("/api/first"),+ _native_request("/missing.png", "image", 404),+ ):+ await callback(native)+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ first = await client.call_tool("browser_network_requests", {"session_id": "browser", "limit": 1})+ assert _listing(first) == {+ "requests": [_request_info(2, "/api/first")],+ "next_cursor": 2,+ "has_more": True,+ "dropped_count": 0,+ }+ await callback(_native_request("/site.css", "stylesheet"))+ await callback(_native_request("/api/last", status=503))+ next_page = await client.call_tool("browser_network_requests", {"session_id": "browser", "after_id": 2})+ assert _listing(next_page) == {+ "requests": [+ _request_info(3, "/missing.png", resource_type="image", status=404),+ _request_info(5, "/api/last", status=503),+ ],+ "next_cursor": 5,+ "has_more": False,+ "dropped_count": 2,+ }+ all_types = await client.call_tool(+ "browser_network_requests", {"session_id": "browser", "include_static": True, "after_id": 2}+ )+ assert _listing(all_types) == {+ "requests": [+ _request_info(3, "/missing.png", resource_type="image", status=404),+ _request_info(4, "/site.css", resource_type="stylesheet"),+ _request_info(5, "/api/last", status=503),+ ],+ "next_cursor": 5,+ "has_more": False,+ "dropped_count": 2,+ }+ filtered = await client.call_tool(+ "browser_network_requests", {"session_id": "browser", "url_pattern": r"/api/"}+ )+ assert _listing(filtered) == {+ "requests": [_request_info(5, "/api/last", status=503)],+ "next_cursor": 5,+ "has_more": False,+ "dropped_count": 2,+ }+ evicted = await client.call_tool("browser_network_request", {"session_id": "browser", "request_id": 2})+ assert isinstance(evicted.content[0], TextContent)+ assert evicted.is_error and "no longer retained" in evicted.content[0].text+ invalid = await client.call_tool("browser_network_requests", {"session_id": "browser", "url_pattern": "["})+ assert isinstance(invalid.content[0], TextContent)+ assert invalid.is_error and "Invalid URL pattern" in invalid.content[0].text+ finally:+ network._detach()++[email protected]("count,after_id,next_cursor", [(0, 0, 0), (2, 0, 2), (2, 1, 2), (2, 2, 2), (2, 9, 9)])+async def test_network_empty_matches_keep_cursor_at_or_after_saved_history(+ count: int, after_id: int, next_cursor: int+) -> None:+ server, network, context = _recording_server()+ try:+ for index in range(count):+ await network._contexts[context]["requestfinished"](_native_request(f"/api/{index}"))+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(+ "browser_network_requests",+ {"session_id": "browser", "url_pattern": "/missing/", "after_id": after_id},+ )+ assert _listing(result) == {+ "requests": [],+ "next_cursor": next_cursor,+ "has_more": False,+ "dropped_count": 0,+ }+ assert network.last_id == count+ finally:+ network._detach()++[email protected]("count,has_more", [(2, False), (3, True)])+async def test_network_listing_limit_checks_for_an_extra_match(count: int, has_more: bool) -> None:+ server, network, context = _recording_server()+ try:+ for index in range(1, count + 1):+ await network._contexts[context]["requestfinished"](_native_request(f"/api/{index}"))+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool("browser_network_requests", {"session_id": "browser", "limit": 2})+ assert _listing(result) == {+ "requests": [_request_info(1, "/api/1"), _request_info(2, "/api/2")],+ "next_cursor": 2,+ "has_more": has_more,+ "dropped_count": 0,+ }+ finally:+ network._detach()+++async def test_network_listing_preserves_url_characters_and_only_returns_summary_fields() -> None:+ server, network, context = _recording_server()+ path = '/api/مرحبا?q="quoted"&path=one\\two#one\ntwo'+ native = _native_request(path)+ native.method = "POST"+ native.post_data_buffer = b'{"secret":"request"}'+ native.headers = {"content-type": "application/json", "cookie": "secret=request"}+ native.all_headers.return_value = native.headers.copy()+ native.existing_response.headers = {"content-type": "application/json", "set-cookie": "secret=response"}+ native.existing_response.all_headers.return_value = native.existing_response.headers.copy()+ native.existing_response.body.return_value = b'{"secret":"response"}'+ await network._contexts[context]["requestfinished"](native)+ network._detach()+ record = network.get(1)+ assert record is not None and record.url == native.url+ assert record.request_body == b'{"secret":"request"}' and record.response.body == b'{"secret":"response"}'+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)+ for reader in readers:+ reader.assert_awaited_once()+ reader.side_effect = AssertionError("Listing saved history must not read the browser")+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ assert _listing(result) == {+ "requests": [_request_info(1, path, "POST")],+ "next_cursor": 1,+ "has_more": False,+ "dropped_count": 0,+ }+ for reader in readers:+ reader.assert_awaited_once()++[email protected]("part", ["summary", "request_headers", "request_body", "response_headers", "response_body"])+async def test_network_detail_returns_only_selected_part(part: Any) -> None:+ server, network = _server()+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 4, "part": part}+ )+ record = network.get(4)+ if part == "summary":+ assert _detail(result) == _detail_info(4, part, _request_info(4, "/api/items", "POST", status=201))+ elif part.endswith("headers"):+ expected = record.response.request_headers if part == "request_headers" else record.response.headers+ assert _detail(result) == _detail_info(4, part, expected)+ else:+ expected = '{"sent":true}' if part == "request_body" else '{"name":"مرحبا"}'+ assert _detail(result) == _detail_info(4, part, expected)+ assert not hasattr(record, "request")+ network.search.assert_not_called()++[email protected]("part", ["request_headers", "response_headers", "response_body"])+def test_saved_response_details_are_local(part: Any) -> None:+ _, network = _server()+ record = network.get(4)+ result = _request_details(record, part)+ assert isinstance(result, NetworkRequestModel)+ if part.endswith("headers"):+ expected = record.response.request_headers if part == "request_headers" else record.response.headers+ assert result.model_dump() == _detail_info(4, part, expected)+ else:+ assert result.model_dump() == _detail_info(4, part, '{"name":"مرحبا"}')++[email protected]("part", ["summary", "request_headers", "response_headers"])+async def test_network_objects_keep_all_summary_fields_and_header_values(part: str) -> None:+ server, network = _server()+ record = network.get(4)+ headers = {+ "id": "007",+ "url": "https://headers.test/",+ "method": "PATCH",+ "resource_type": "xhr",+ "status": "0200",+ "x-extra": "kept",+ }+ if part == "request_headers":+ record.request_headers = headers+ elif part == "response_headers":+ record.response_headers = headers+ expected = _request_info(4, "/api/items", "POST", status=201) if part == "summary" else headers+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(+ "browser_network_request",+ {"session_id": "browser", "request_id": 4, "part": part},+ )+ assert _detail(result) == _detail_info(4, part, expected)++[email protected]("part", ["request_body", "response_body"])+async def test_network_unicode_body_returns_the_full_original_text(part: str) -> None:+ server, network = _server()+ record = network.get(4)+ text = 'Aمرحبا🙂e\u0301\n"z"'+ if part == "request_body":+ record.request_body = text.encode()+ else:+ record.response._raw_body = text.encode()+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 4, "part": part}+ )+ assert _detail(result) == _detail_info(4, part, text)++[email protected]("part", ["request_body", "response_body"])[email protected]("size", [0, 10001, 50001, 100003])+async def test_network_detail_returns_complete_empty_and_large_saved_bodies(part: str, size: int) -> None:+ server, network, context = _recording_server()+ native = _native_request("/api/full-body")+ text = 'start\n"' + "x" * (size - 11) + '"end' if size else ""+ assert len(text) == size+ if part == "request_body":+ native.post_data_buffer = text.encode()+ native.headers = {"content-type": "text/plain"}+ native.all_headers.return_value = native.headers.copy()+ else:+ native.existing_response.body.return_value = text.encode()+ await network._contexts[context]["requestfinished"](native)+ network._detach()+ record = network.get(1)+ assert record is not None+ assert (record.request_body if part == "request_body" else record.response.body) == text.encode()+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)+ for reader in readers:+ reader.assert_awaited_once()+ reader.side_effect = AssertionError("Reading full saved bodies must not read the browser")+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}+ )+ assert _detail(result) == _detail_info(1, part, text)+ for reader in readers:+ reader.assert_awaited_once()++[email protected]("case", ["empty", "binary", "unavailable", "too_large"])+async def test_network_detail_body_availability(case: str) -> None:+ server, network = _server()+ record = network.get(4)+ if case == "empty":+ record.response.status = 204+ record.response._raw_body = b""+ elif case == "binary":+ record.response.headers["content-type"] = "image/png"+ elif case == "too_large":+ record.response.meta["body_note"] = "too large; not saved."+ else:+ record.response.meta["body_note"] = "Context closed"+ result = await server.browser_network_request("browser", 4, "response_body")+ expected = {+ "empty": _detail_info(4, "response_body", ""),+ "binary": _detail_info(4, "response_body", note="Only text bodies can be displayed."),+ "unavailable": _detail_info(4, "response_body", note="Context closed"),+ "too_large": _detail_info(4, "response_body", note="too large; not saved."),+ }+ assert result.model_dump() == expected[case]++[email protected]("part", ["request_headers", "response_headers"])[email protected]("partial", [False, True])+async def test_network_detail_saved_headers_report_completeness(part: Any, partial: bool) -> None:+ server, network = _server()+ record = network.get(4)+ headers = record.request_headers if part == "request_headers" else record.response_headers+ key = "request_headers_partial" if part == "request_headers" else "headers_partial"+ record.response.meta[key] = partial+ result = await server.browser_network_request("browser", 4, part)+ assert result.model_dump() == _detail_info(+ 4, part, headers, note="Headers are partial; full headers unavailable." if partial else None+ )+++ "content_type, body, encoding, expected",+ [+ ("text/plain; charset = iso-8859-1", b"caf\xe9", "iso-8859-1", "café"),+ ("text/plain; charset=invalid-codec", b"hi", "invalid-codec", "hi"),+ ('text/plain; CHARSET = "ISO-8859-1"', b"caf\xe9", "iso-8859-1", "café"),+ ("text/plain; charset = 'ISO-8859-1'", b"caf\xe9", "iso-8859-1", "café"),+ ("text/plain; charset=iso-8859-1", "café".encode(), "utf-8", "café"),+ ("application/problem+json", b'{"error":true}', "utf-8", '{"error":true}'),+ ("application/xml", b"<ok/>", "utf-8", "<ok/>"),+ ],+)+async def test_network_detail_decodes_text_for_any_resource(+ content_type: str, body: bytes, encoding: str, expected: str+) -> None:+ server, network = _server()+ record = network.get(4)+ record.resource_type = "document"+ record.response.headers["content-type"] = content_type+ record.response._raw_body = body+ record.response.encoding = encoding+ result = await server.browser_network_request("browser", 4, "response_body")+ assert result.model_dump() == _detail_info(4, "response_body", expected)+++ "response_type,native_body,response_encoding,response_text,request_type,request_body,request_text",+ [+ (+ "text/plain; charset=iso-8859-15",+ "café ¤".encode(),+ "utf-8",+ "café ¤",+ "text/plain; charset=iso-8859-15",+ b"caf\xe9 \xa4",+ "café €",+ ),+ (+ "text/plain; charset=iso-8859-1",+ b"caf\xe9",+ "iso-8859-1",+ "café",+ "text/plain; charset=iso-8859-1",+ b"caf\xe9",+ "café",+ ),+ (+ "text/plain; charset=invalid-codec",+ "café".encode(),+ "utf-8",+ "café",+ "text/plain; charset=invalid-codec",+ "café".encode(),+ "café",+ ),+ (+ "text/plain; charset=invalid-codec",+ b"caf\xe9",+ "utf-8",+ "caf\ufffd",+ "text/plain; charset=invalid-codec",+ b"caf\xe9",+ "caf\ufffd",+ ),+ ],+)+async def test_network_response_uses_saved_encoding_and_request_uses_its_header(+ response_type: str,+ native_body: bytes,+ response_encoding: str,+ response_text: str,+ request_type: str,+ request_body: bytes,+ request_text: str,+) -> None:+ server, network, context = _recording_server()+ native = _native_request("/api/encoded")+ native.method = "POST"+ native.headers = {"content-type": request_type}+ native.all_headers.return_value = native.headers.copy()+ native.post_data_buffer = request_body+ native.existing_response.headers = {"content-type": response_type}+ native.existing_response.all_headers.return_value = native.existing_response.headers.copy()+ native.existing_response.body.return_value = native_body+ await network._contexts[context]["requestfinished"](native)+ network._detach()+ record = network.get(1)+ assert record is not None+ assert record.response.encoding == response_encoding+ assert record.response.body == native_body and record.request_body == request_body+ assert record.response_headers == {"content-type": response_type}+ assert record.request_headers == {"content-type": request_type}+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)+ for reader in readers:+ reader.assert_awaited_once()+ reader.side_effect = AssertionError("Saved history must not read the browser")+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ for part, expected in (("response_body", response_text), ("request_body", request_text)):+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}+ )+ assert _detail(result) == _detail_info(1, part, expected)+ for part, expected in (("response_headers", response_type), ("request_headers", request_type)):+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}+ )+ assert _detail(result) == _detail_info(1, part, {"content-type": expected})+ for reader in readers:+ reader.assert_awaited_once()+ assert record.response.body == native_body and record.request_body == request_body+++async def test_network_request_body_absent_and_binary() -> None:+ server, network = _server()+ record = network.get(4)+ record.request_body = None+ result = await server.browser_network_request("browser", 4, "request_body")+ assert result.model_dump() == _detail_info(4, "request_body", note="Body is absent.")+ record.request_body = b"binary"+ record.request_headers["content-type"] = "application/octet-stream"+ record.response.request_headers["content-type"] = "application/octet-stream"+ result = await server.browser_network_request("browser", 4, "request_body")+ assert result.model_dump() == _detail_info(4, "request_body", note="Only text bodies can be displayed.")+++ "part,body,content_type,status,expected,note",+ [+ ("request_body", None, "text/plain", 200, None, "Body is absent."),+ ("request_body", b"", "text/plain", 200, "", None),+ ("request_body", b"", None, 200, "", None),+ ("request_body", b"binary", "application/octet-stream", 200, None, "Only text bodies can be displayed."),+ ("response_body", b"", "text/plain", 200, "", None),+ ("response_body", b"", "application/json", 204, "", None),+ ("response_body", b"", None, 200, None, "Non-text body; not saved."),+ ("response_body", b"", None, 204, "", None),+ ("response_body", b"binary", "image/png", 200, None, "Non-text body; not saved."),+ ("response_body", b"file", "application/pdf", 200, None, "Non-text body; not saved."),+ ("response_body", b"file", "application/octet-stream", 200, None, "Non-text body; not saved."),+ ("response_body", b"saved", "application/unknown", 200, None, "Non-text body; not saved."),+ ("response_body", b"saved", None, 200, None, "Non-text body; not saved."),+ ("response_body", b'{"ok":true}', "application/problem+json", 200, '{"ok":true}', None),+ ],+)+async def test_network_reads_saved_parts_without_native_access(+ part: str, body: bytes | None, content_type: str | None, status: int, expected: str | None, note: str | None+) -> None:+ server, network, context = _recording_server()+ native = _native_request("/api/saved", status=status)+ headers = {"content-type": content_type} if content_type is not None else {}+ if part == "request_body":+ native.post_data_buffer = body+ native.headers = headers+ native.all_headers.return_value = headers.copy()+ else:+ native.existing_response.body.return_value = body+ native.existing_response.headers = headers+ native.existing_response.all_headers.return_value = headers.copy()+ await network._contexts[context]["requestfinished"](native)+ network._detach()+ record = network.get(1)+ assert record is not None+ skipped = part == "response_body" and note == "Non-text body; not saved."+ assert record.response.meta == ({"body_note": note} if skipped else {})+ if part == "response_body":+ assert record.response.body == (b"" if skipped else body)+ assert native.existing_response.body.await_count == (not skipped and status != 204)+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)+ counts = [reader.await_count for reader in readers]+ for reader in readers:+ reader.side_effect = AssertionError("Saved history must not read the browser")+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}+ )+ assert _detail(result) == _detail_info(1, part, expected, note=note)+ for saved_part in ("summary", "request_headers", "response_headers"):+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": saved_part}+ )+ expected_data = (+ _request_info(1, "/api/saved", status=status)+ if saved_part == "summary"+ else record.request_headers+ if saved_part == "request_headers"+ else record.response_headers+ )+ assert _detail(result) == _detail_info(1, saved_part, expected_data)+ assert [reader.await_count for reader in readers] == counts+++ "error,expected",+ [+ (None, "too large; not saved."),+ (RuntimeError("Context closed"), "Context closed"),+ (RuntimeError(), "could not be saved."),+ ],+)+async def test_network_body_note_does_not_replace_other_saved_parts(error: Exception | None, expected: str) -> None:+ server, network, context = _recording_server()+ native = _native_request("/api/body-note")+ native.method = "POST"+ native.post_data_buffer = b'{"sent":true}'+ native.headers = {"content-type": "application/json"}+ native.all_headers.return_value = native.headers.copy()+ response_headers = {"content-type": "application/json", "x-recorded": "yes"}+ if error is None:+ response_headers["content-length"] = str(1024 * 1024 + 1)+ else:+ native.existing_response.body.side_effect = error+ native.existing_response.headers = response_headers+ native.existing_response.all_headers.return_value = response_headers.copy()+ await network._contexts[context]["requestfinished"](native)+ network._detach()+ record = network.get(1)+ assert record is not None and record.response.meta == {"body_note": expected}+ assert record.response.body == b"" and record.request_body == b'{"sent":true}'+ assert native.existing_response.body.await_count == (error is not None)+ readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)+ counts = [reader.await_count for reader in readers]+ for reader in readers:+ reader.side_effect = AssertionError("Saved history must not read the browser")+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ for part in ("response_body", "summary", "request_headers", "response_headers", "request_body"):+ result = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": 1, "part": part}+ )+ if part == "response_body":+ assert _detail(result) == _detail_info(1, part, note=expected)+ elif part == "summary":+ assert _detail(result) == _detail_info(1, part, _request_info(1, "/api/body-note", "POST"))+ elif part.endswith("headers"):+ assert _detail(result) == _detail_info(+ 1, part, native.headers if part == "request_headers" else response_headers+ )+ else:+ assert _detail(result) == _detail_info(1, part, '{"sent":true}')+ assert [reader.await_count for reader in readers] == counts+++ "name, options",+ [+ ("browser_network_requests", {"after_id": -1}),+ ("browser_network_requests", {"limit": 0}),+ ("browser_network_requests", {"limit": 201}),+ ("browser_network_request", {"request_id": 0}),+ ("browser_network_request", {}),+ ("browser_network_request", {"request_id": 4, "part": "all"}),+ ],+)+async def test_network_invalid_input_does_not_read_history(name: str, options: dict[str, Any]) -> None:+ server, network = _server()+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool(name, {"session_id": "browser", **options})+ assert result.is_error+ network.get.assert_not_called()+ network.search.assert_not_called()++[email protected]("case", ["missing", "static", "dead"])[email protected]("name", ["browser_network_requests", "browser_network_request"])+async def test_network_session_guards(case: str, name: str) -> None:+ server, network = _server()+ if case == "missing":+ server._sessions.clear()+ elif case == "static":+ server._sessions["browser"].session_type = "static"+ else:+ server._sessions["browser"].session._is_alive = False+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ options = {"request_id": 4} if name == "browser_network_request" else {}+ assert (await client.call_tool(name, {"session_id": "browser", **options})).is_error+ network.get.assert_not_called()+ network.search.assert_not_called()+++async def test_network_missing_id_and_search_error_reach_client() -> None:+ server, network = _server()+ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ result = await client.call_tool("browser_network_request", {"session_id": "browser", "request_id": 99})+ assert isinstance(result.content[0], TextContent)+ assert result.is_error and "no longer retained" in result.content[0].text+ network.search.side_effect = ValueError("Invalid URL regular expression")+ result = await client.call_tool("browser_network_requests", {"session_id": "browser", "url_pattern": "["})+ assert isinstance(result.content[0], TextContent)+ assert result.is_error and "regular expression" in result.content[0].text+++async def test_network_live_capture_actions_navigation_and_reads() -> None:+ server = ScraplingMCPServer(executable_path=getenv("SCRAPLING_EXECUTABLE_PATH"))+ seen: list[str] = []+ html = """<html><head><link rel="icon" href="data:,"></head><body><button id="load" onclick="fetch('/api/items', {method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({sent: true})}).then(r => r.json()).then(() => document.body.dataset.done='yes')">Load</button></body></html>"""+ responses = {+ "/api/items": (200, "application/json", b'{"items":[1,2]}'),+ "/api/empty": (200, "text/plain", b""),+ "/api/absent": (204, "application/json", b""),+ "/api/binary": (200, "application/octet-stream", b"\x00\xff\x80binary"),+ "/api/error": (503, "application/json", b'{"error":"unavailable"}'),+ }++ async def serve(route: Any) -> None:+ seen.append(route.request.url)+ path = urlsplit(route.request.url).path+ if path == "/api/failed":+ await route.abort("failed")+ return+ status, content_type, body = responses.get(path, (200, "text/html", html.encode()))+ await route.fulfill(status=status, content_type=content_type, body=body, headers={"x-recorded": "yes"})++ async def captured(session: Any, pattern: str) -> NetworkRequest:+ while not (records := session.network.search(url_pattern=pattern)):+ await sleep(0.01)+ return records[0]++ async with Client(server._build_server("127.0.0.1", 8000)) as client:+ opened = await client.call_tool("browser_open", {"session_id": "browser"})+ assert not opened.is_error+ session = server._sessions["browser"].session+ try:+ assert session.network.enabled and opened.structured_content["settings"]["record_requests"] is True+ empty = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ assert _listing(empty) == {"requests": [], "next_cursor": 0, "has_more": False, "dropped_count": 0}+ await session.context.route("**/*", serve)+ assert not (+ await client.call_tool("browser_fetch", {"session_id": "browser", "url": "https://network.test/"})+ ).is_error+ acted = await client.call_tool(+ "browser_actions",+ {+ "session_id": "browser",+ "actions": [+ {"type": "click", "selector": "#load"},+ {"type": "wait_element", "selector": "body[data-done=yes]", "timeout": 2000},+ ],+ },+ )+ assert not acted.is_error+ request = await wait_for(captured(session, "/api/items$"), timeout=5)+ extra = await client.call_tool(+ "browser_evaluate",+ {+ "session_id": "browser",+ "expression": "async () => { await Promise.allSettled(['/api/empty', '/api/absent', '/api/binary', '/api/error', '/api/failed'].map(path => fetch(path).then(response => response.arrayBuffer()))); return true; }",+ },+ )+ assert not extra.is_error+ records = {+ name: await wait_for(captured(session, f"/api/{name}$"), timeout=5)+ for name in ("empty", "absent", "binary", "error")+ }+ assert session.network.search(url_pattern="/api/failed$") == []+ for name, record in records.items():+ assert record.response.body == (b"" if name == "binary" else responses[f"/api/{name}"][2])+ assert record.response.meta == ({"body_note": "Non-text body; not saved."} if name == "binary" else {})+ before = list(seen)+ page = session.page_pool.get_ready_page()+ assert page is not None+ try:+ listed = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ detail = await client.call_tool(+ "browser_network_request",+ {"session_id": "browser", "request_id": request.id, "part": "response_body"},+ )+ assert not listed.is_error and not detail.is_error+ assert request.response is not None and request.response.body == b'{"items":[1,2]}'+ expected_requests = sorted(+ [_request_info(request.id, "/api/items", "POST")]+ + [+ _request_info(record.id, f"/api/{name}", status=responses[f"/api/{name}"][0])+ for name, record in records.items()+ ],+ key=lambda item: item["id"],+ )+ expected_listing = {+ "requests": expected_requests,+ "next_cursor": expected_requests[-1]["id"],+ "has_more": False,+ "dropped_count": 0,+ }+ assert _listing(listed) == expected_listing+ assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')+ request_body = await client.call_tool(+ "browser_network_request",+ {"session_id": "browser", "request_id": request.id, "part": "request_body"},+ )+ assert _detail(request_body) == _detail_info(request.id, "request_body", '{"sent":true}')+ for name, expected, note in (+ ("empty", "", None),+ ("absent", "", None),+ ("binary", None, "Non-text body; not saved."),+ ("error", '{"error":"unavailable"}', None),+ ):+ result = await client.call_tool(+ "browser_network_request",+ {"session_id": "browser", "request_id": records[name].id, "part": "response_body"},+ )+ assert _detail(result) == _detail_info(records[name].id, "response_body", expected, note=note)+ headers = await client.call_tool(+ "browser_network_request",+ {"session_id": "browser", "request_id": records[name].id, "part": "response_headers"},+ )+ assert _detail(headers) == _detail_info(+ records[name].id, "response_headers", records[name].response_headers+ )+ assert _detail(headers)["data"]["x-recorded"] == "yes"+ assert page.state == "busy" and seen == before+ finally:+ page.mark_ready()+ assert not (+ await client.call_tool("browser_fetch", {"session_id": "browser", "url": "https://network.test/next"})+ ).is_error+ await wait_for(captured(session, "/next$"), timeout=5)+ listed = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ assert _listing(listed) == expected_listing+ await session.close_pages()+ before = list(seen)+ detail = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}+ )+ assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')+ for record in records.values():+ saved = await client.call_tool(+ "browser_network_request",+ {"session_id": "browser", "request_id": record.id, "part": "response_headers"},+ )+ assert _detail(saved) == _detail_info(record.id, "response_headers", record.response_headers)+ assert _detail(saved)["data"]["x-recorded"] == "yes"+ assert seen == before+ finally:+ try:+ assert not (await client.call_tool("close_session", {"session_id": "browser"})).is_error+ finally:+ await session.close()+ assert not session.network._contexts+ assert request.response.body == b'{"items":[1,2]}'+ closed = await client.call_tool("browser_network_requests", {"session_id": "browser"})+ assert closed.is_error+ closed_detail = await client.call_tool(+ "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}+ )+ assert closed_detail.is_errordiff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.pynew file mode 100644index 0000000..45dca2a--- /dev/null+++ b/tests/fetchers/test_network_browser.py@@ -0,0 +1,952 @@+import asyncio+from base64 import b64decode+from contextlib import suppress+from dataclasses import dataclass, field+from gzip import compress+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from json import loads+from os import getenv+from socket import SHUT_RDWR+from threading import Event, Thread+from time import monotonic+from unittest.mock import Mock+from urllib.parse import urlsplit++import pytest+from patchright.async_api import Response as AsyncPatchrightResponse+from patchright.sync_api import Response as PatchrightResponse+from playwright.async_api import Response as AsyncPlaywrightResponse+from playwright.sync_api import Response as PlaywrightResponse++from scrapling.core._types import Any, Generator+from scrapling.core._network_formatting import _request_details+from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator+++HTML = b"""<!DOCTYPE html><html><head><link rel="icon" href="data:,"><link rel="stylesheet" href="/style.css"></head>+<body><img src="/image.png"><script src="/script.js"></script></body></html>"""+SCRIPT = b"""Promise.all([+fetch('/api/initial').then(response => response.json()),+new Promise(resolve => { const xhr = new XMLHttpRequest(); xhr.open('GET', '/api/xhr');+xhr.onload = () => resolve(JSON.parse(xhr.responseText)); xhr.send(); })+]).then(() => document.body.dataset.initial = 'ready');"""+LATER = """async () => {+ await fetch('/api/post', {method: 'POST', headers: {'Content-Type': 'application/json', 'X-Sent': 'request'}, body: '{"sent":true}'}).then(response => response.json());+ await fetch('/api/repeated').then(response => response.text());+ await fetch('/api/repeated').then(response => response.text());+ await fetch('/api/status').then(response => response.text());+ await fetch('/redirect').then(response => response.text());+ await fetch('/api/fail').catch(() => null);+ await fetch('/api/large').then(response => response.text());+}"""+PENDING = """() => {+ fetch('/api/pending').then(response => response.text()).then(() => document.body.dataset.pending = 'done');+}"""+BLANK = b'<!DOCTYPE html><html><head><link rel="icon" href="data:,"></head><body>ready</body></html>'+LARGE = b"x" * (1024 * 1024 + 1)+BINARY = bytes(range(256))+PNG = b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aZ1cAAAAASUVORK5CYII=")+TEXT_RESPONSES = {+ "/mime/json": ("application/json", b'{"saved":true}'),+ "/mime/xml": ("application/xml", b"<saved>true</saved>"),+ "/mime/vendor-json": ("application/vnd.scrapling+json", b'{"saved":true}'),+ "/mime/vendor-xml": ("application/vnd.scrapling+xml", b"<saved>true</saved>"),+ "/mime/javascript": ("application/javascript", b"window.saved = true;"),+ "/mime/graphql": ("application/graphql", b"{ saved }"),+ "/mime/form": ("application/x-www-form-urlencoded", b"saved=true"),+}+MIME_RESPONSES = {+ **TEXT_RESPONSES,+ "/mime/image": ("image/png", PNG),+ "/mime/pdf": ("application/pdf", b"%PDF-1.7\n%%EOF"),+ "/mime/file": ("application/octet-stream", BINARY),+ "/mime/media": ("audio/wav", b"RIFF\x04\x00\x00\x00WAVE"),+ "/mime/font": ("font/woff2", b"wOF2\x00\x01\x00\x00"),+ "/mime/missing": (None, b'{"saved":false}'),+ "/mime/unknown": ("application/x-scrapling-unknown", b'{"saved":false}'),+}+MIME_FETCH = """async paths => {+ const bodies = {};+ for (const path of paths) {+ const response = await fetch(path);+ bodies[path] = Array.from(new Uint8Array(await response.arrayBuffer()));+ }+ return bodies;+}"""+CHARSET_BODY = "caf\u00e9 \u20ac".encode("iso-8859-15")+CHARSET_PATHS = ("/wire/charset", "/wire/charset-spaced", "/wire/raw-charset")+WIRE_PATHS = ["/wire/chunked", "/wire/gzip", "/wire/binary", *CHARSET_PATHS, "/wire/head", "/wire/empty"]+WIRE_FETCH = """async paths => {+ const bodies = {};+ for (const path of paths) {+ const response = await fetch(path, {method: path.endsWith('/head') ? 'HEAD' : 'GET'});+ const body = await response.arrayBuffer();+ if (path.includes('charset') || path.endsWith('/binary')) bodies[path] = Array.from(new Uint8Array(body));+ }+ return bodies;+}"""+++@dataclass+class _Gate:+ started: Event = field(default_factory=Event)+ release: Event = field(default_factory=Event)+ finished: Event = field(default_factory=Event)+++class _Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, format: str, *args: Any) -> None:+ pass++ def do_GET(self) -> None:+ path = urlsplit(self.path).path+ if path == "/api/fail":+ self.connection.shutdown(SHUT_RDWR)+ self.connection.close()+ return+ if path == "/api/pending":+ if not getattr(self.server, "release_pending").wait(10):+ getattr(self.server, "errors").append("Timed out waiting to release /api/pending")+ return+ status, body = 200, b'{"ok":true}'+ content_type: str | None = "application/json"+ if path in ("/", "/second", "/fresh", "/other"):+ content_type, body = "text/html", HTML+ elif path == "/blank":+ content_type, body = "text/html", BLANK+ elif path == "/style.css":+ content_type, body = "text/css", b"body { color: black; }"+ elif path == "/script.js":+ content_type, body = "application/javascript", SCRIPT+ elif path == "/image.png":+ content_type, body = "image/png", PNG+ elif path == "/api/status":+ status, content_type, body = 503, "text/plain", b"Try later"+ elif path == "/api/large":+ content_type, body = "text/plain", LARGE+ elif path == "/redirect":+ status, body = 302, b""+ elif path in {"/wire/chunked", "/wire/gzip"}:+ content_type, body = "text/plain", compress(LARGE) if path.endswith("gzip") else LARGE+ elif path == "/wire/binary":+ content_type, body = "application/octet-stream", BINARY+ elif path == "/wire/charset":+ content_type, body = "text/plain; charset=iso-8859-15", CHARSET_BODY+ elif path == "/wire/charset-spaced":+ content_type, body = 'text/plain; CHARSET = "iso-8859-15"', CHARSET_BODY+ elif path == "/wire/raw-charset":+ content_type, body = 'application/octet-stream; CHARSET = "iso-8859-15"', CHARSET_BODY+ elif path == "/wire/head":+ content_type = "application/octet-stream"+ elif path == "/wire/empty":+ status, content_type, body = 204, "application/octet-stream", b""+ elif path in MIME_RESPONSES:+ content_type, body = MIME_RESPONSES[path]+ self.send_response(status)+ if content_type is not None:+ self.send_header("Content-Type", content_type)+ self.send_header("Transfer-Encoding", "chunked") if path == "/wire/chunked" else self.send_header(+ "Content-Length", str(len(body))+ )+ self.send_header("X-Recorded", "response")+ if path == "/redirect":+ self.send_header("Location", "/redirected")+ if path == "/wire/gzip":+ self.send_header("Content-Encoding", "gzip")+ self.end_headers()+ gate = getattr(self.server, "gates").get(path)+ try:+ if gate is not None:+ self.wfile.write(body[:1])+ self.wfile.flush()+ gate.started.set()+ if not gate.release.wait(10):+ getattr(self.server, "errors").append(f"Timed out waiting to release {path}")+ return+ body = body[1:]+ if path == "/wire/chunked":+ for start in range(0, len(body), 65536):+ chunk = body[start : start + 65536]+ self.wfile.write(f"{len(chunk):x}\r\n".encode() + chunk + b"\r\n")+ self.wfile.write(b"0\r\n\r\n")+ elif self.command != "HEAD":+ self.wfile.write(body)+ except (BrokenPipeError, ConnectionResetError):+ pass+ finally:+ if gate is not None:+ gate.finished.set()++ def do_HEAD(self) -> None:+ self.do_GET()++ def do_POST(self) -> None:+ body = self.rfile.read(int(self.headers.get("Content-Length", 0)))+ self.send_response(201)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(body)))+ self.send_header("X-Recorded", "response")+ self.end_headers()+ self.wfile.write(body)++[email protected](scope="module")+def release_pending() -> Event:+ return Event()++[email protected](scope="module")+def network_gates() -> dict[str, _Gate]:+ return {}++[email protected](scope="module")+def network_url(release_pending: Event, network_gates: dict[str, _Gate]) -> Generator[str, None, None]:+ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ setattr(server, "release_pending", release_pending)+ setattr(server, "gates", network_gates)+ errors: list[str] = []+ setattr(server, "errors", errors)+ thread = Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_port}"+ finally:+ release_pending.set()+ for gate in network_gates.values():+ gate.release.set()+ server.shutdown()+ server.server_close()+ thread.join(timeout=5)+ assert not thread.is_alive()+ assert not errors+++def _options(**kwargs: Any) -> dict[str, Any]:+ return {+ "executable_path": getenv("SCRAPLING_EXECUTABLE_PATH"),+ "google_search": False,+ "record_requests": True,+ "retries": 1,+ **kwargs,+ }++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_disabled_by_default(session_type: Any) -> None:+ session = session_type()+ context = Mock()+ session._initialize_context(session._config, context)+ assert not session.network.enabled+ context.on.assert_not_called()+ assert session.network.search() == [] and session.network.last_id == 0+ assert not session.network._contexts+ session.close()++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_disabled_by_default(session_type: Any) -> None:+ session = session_type()+ context = Mock()+ await session._initialize_context(session._config, context)+ assert not session.network.enabled+ context.on.assert_not_called()+ assert session.network.search() == [] and session.network.last_id == 0+ assert not session.network._contexts+ await session.close()+++def _wait_saved(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:+ for _ in range(250):+ records = network.search(url_pattern=pattern, after_id=after_id)+ if len(records) >= count:+ return records+ page.wait_for_timeout(20)+ raise AssertionError(f"Expected {count} saved responses for {pattern!r}, got {len(records)}")+++async def _wait_saved_async(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:+ for _ in range(250):+ records = network.search(url_pattern=pattern, after_id=after_id)+ if len(records) >= count:+ return records+ await asyncio.sleep(0.02)+ raise AssertionError(f"Expected {count} saved responses for {pattern!r}, got {len(records)}")+++def _check_initial(network: Any, response: Any) -> Any:+ records = network.search()+ assert network.enabled+ assert {record.resource_type for record in records} >= {"document", "script", "stylesheet", "image", "fetch", "xhr"}+ assert len({record.id for record in records}) == len(records)+ initial = network.search(url_pattern=r"/api/initial$", resource_type="fetch")+ assert len(initial) == 1+ assert initial[0].status == 200+ assert initial[0].request_body is None+ assert initial[0].response.json() == {"ok": True}+ assert "body_note" not in initial[0].response.meta and not hasattr(initial[0], "request")+ assert {urlsplit(item.url).path for item in response.captured_xhr} == {"/api/initial", "/api/xhr"}+ assert all(item.json() == {"ok": True} for item in response.captured_xhr)+ image = network.search(url_pattern=r"/image\.png$")[0]+ assert image.response.body == b"" and image.response.meta.get("body_note") == "Non-text body; not saved."+ for path, body in ((r"/script\.js$", SCRIPT), (r"/style\.css$", b"body { color: black; }")):+ asset = network.search(url_pattern=path)[0]+ assert asset.response.body == body and "body_note" not in asset.response.meta+ filtered = network.search(include_static=False)+ assert filtered and all(record.resource_type in ("document", "xhr", "fetch") for record in filtered)+ return initial[0]+++def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:+ records = network.search(after_id=after_id)+ assert records and all(record.id > after_id for record in records)+ post = network.search(method="POST", status=201)+ assert len(post) == 1+ assert loads(post[0].request_body) == {"sent": True}+ duplicate = network.search(url_pattern=r"/api/repeated$")+ assert len(duplicate) == 2 and duplicate[0].id != duplicate[1].id+ assert len(network.search(url_pattern=r"/api/repeated$", limit=1)) == 1+ status = network.search(status=503)+ assert len(status) == 1+ assert not network.search(url_pattern=r"/api/fail$")+ redirect = network.search(url_pattern=r"/redirect$")+ assert len(redirect) == 1 and redirect[0].status == 302+ redirected = network.search(url_pattern=r"/redirected$", status=200)+ assert len(redirected) == 1 and redirected[0].id != redirect[0].id+ assert redirect[0].response_headers["location"] == "/redirected"+ return post[0], status[0], network.search(url_pattern=r"/api/large$")[0]++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:+ with session_type(+ **_options(+ capture_xhr=r"/api/",+ page_action=lambda page: page.locator("body[data-initial=ready]").wait_for(state="attached"),+ )+ ) as session:+ response = session.fetch(network_url + "/")+ page = session.page_pool.pages[0].page+ _wait_saved(page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6)+ initial = _check_initial(session.network, response)+ assert initial.response.json() == {"ok": True}+ page = session.page_pool.pages[0].page+ after_id = session.network.last_id+ with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):+ page.evaluate(LATER)+ _wait_saved(page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7)+ post, status, large = _check_later(session.network, after_id)+ assert post.response.request_headers["x-sent"] == "request"+ assert post.response.headers["x-recorded"] == "response"+ assert post.response.json() == {"sent": True}+ assert status.response.body == b"Try later"+ assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""+ release_pending.clear()+ try:+ with page.expect_request_finished(lambda request: request.url.endswith("/api/pending")):+ with page.expect_request("**/api/pending"):+ page.evaluate(PENDING)+ assert not session.network.search(url_pattern=r"/api/pending$")+ release_pending.set()+ pending = _wait_saved(page, session.network, r"/api/pending$")[0]+ assert pending.status == 200 and pending.response.json() == {"ok": True}+ finally:+ release_pending.set()+ other = session.context.new_page()+ try:+ other.goto(network_url + "/other")+ other.locator("body[data-initial=ready]").wait_for(state="attached")+ _wait_saved(other, session.network, r"/other$")+ finally:+ other.close()+ assert session.network.search(url_pattern=r"/other$", resource_type="document")+ session.fetch(network_url + "/second")+ assert session.page_pool.pages[0].page is page+ session.close_pages()+ assert page.is_closed() and session.network.get(post.id).url == post.url+ fresh_id = session.network.last_id+ session.fetch(network_url + "/fresh")+ _wait_saved(session.page_pool.pages[0].page, session.network, r"/(?:fresh|api/initial)$", 2, fresh_id)+ assert session.page_pool.pages[0].page is not page+ assert session.network.search(url_pattern=r"/fresh$", resource_type="document")+ unread = session.network.search(url_pattern=r"/api/initial$")[-1]+ page = session.page_pool.pages[0].page+ with page.expect_request_finished(lambda request: request.url.endswith("/api/closing")):+ page.evaluate("fetch('/api/closing', {method: 'POST', body: 'closing'}).then(response => response.text())")+ _wait_saved(page, session.network, r"/api/closing$")+ last_id = session.network.last_id+ assert session.network.last_id == last_id+ assert session.network.get(post.id).status == 201+ assert session.network.get(post.id).method == "POST"+ closing = session.network.search(url_pattern=r"/api/closing$")[0]+ assert closing.response.body == b"closing" and "body_note" not in closing.response.meta+ assert unread.response.json() == {"ok": True}+ assert initial.response.json() == {"ok": True}+ assert post.response.json() == {"sent": True}+ assert post.response.request_headers["x-sent"] == "request"+ assert post.response.headers["x-recorded"] == "response"++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:+ async def initial_page(page: Any) -> None:+ await page.locator("body[data-initial=ready]").wait_for(state="attached")++ async with session_type(**_options(capture_xhr=r"/api/", page_action=initial_page)) as session:+ response = await session.fetch(network_url + "/")+ page = session.page_pool.pages[0].page+ await _wait_saved_async(+ page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6+ )+ initial = _check_initial(session.network, response)+ assert initial.response.json() == {"ok": True}+ page = session.page_pool.pages[0].page+ after_id = session.network.last_id+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):+ await page.evaluate(LATER)+ await _wait_saved_async(+ page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7+ )+ post, status, large = _check_later(session.network, after_id)+ assert post.response.request_headers["x-sent"] == "request"+ assert post.response.headers["x-recorded"] == "response"+ assert post.response.json() == {"sent": True}+ assert status.response.body == b"Try later"+ assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""+ release_pending.clear()+ try:+ async with page.expect_request("**/api/pending"):+ await page.evaluate(PENDING)+ assert not session.network.search(url_pattern=r"/api/pending$")+ release_pending.set()+ pending = (await _wait_saved_async(page, session.network, r"/api/pending$"))[0]+ assert pending.status == 200 and pending.response.json() == {"ok": True}+ assert not any("/api/pending" in record.url for record in response.captured_xhr)+ finally:+ release_pending.set()+ other = await session.context.new_page()+ try:+ await other.goto(network_url + "/other")+ await other.locator("body[data-initial=ready]").wait_for(state="attached")+ await _wait_saved_async(other, session.network, r"/other$")+ finally:+ await other.close()+ assert session.network.search(url_pattern=r"/other$", resource_type="document")+ await session.fetch(network_url + "/second")+ assert session.page_pool.pages[0].page is page+ await session.close_pages()+ assert page.is_closed() and session.network.get(post.id).url == post.url+ fresh_id = session.network.last_id+ await session.fetch(network_url + "/fresh")+ await _wait_saved_async(+ session.page_pool.pages[0].page, session.network, r"/(?:fresh|api/initial)$", 2, fresh_id+ )+ assert session.page_pool.pages[0].page is not page+ assert session.network.search(url_pattern=r"/fresh$", resource_type="document")+ unread = session.network.search(url_pattern=r"/api/initial$")[-1]+ page = session.page_pool.pages[0].page+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/closing")):+ await page.evaluate(+ "fetch('/api/closing', {method: 'POST', body: 'closing'}).then(response => response.text())"+ )+ await _wait_saved_async(page, session.network, r"/api/closing$")+ last_id = session.network.last_id+ assert session.network.last_id == last_id+ assert session.network.get(post.id).status == 201+ assert session.network.get(post.id).method == "POST"+ closing = session.network.search(url_pattern=r"/api/closing$")[0]+ assert closing.response.body == b"closing" and "body_note" not in closing.response.meta+ assert unread.response.json() == {"ok": True}+ assert initial.response.json() == {"ok": True}+ assert post.response.json() == {"sent": True}+ assert post.response.request_headers["x-sent"] == "request"+ assert post.response.headers["x-recorded"] == "response"+++def _check_eviction(network: Any) -> int:+ records = network.search()+ assert len(records) == 2 and network.dropped_count > 0+ assert network.get(1) is None+ last_id = network.last_id+ network.clear()+ assert network.search() == [] and network.last_id == last_id+ return last_id++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:+ with session_type(**_options(max_recorded_requests=2)) as session:+ session.fetch(network_url + "/")+ page = session.page_pool.pages[0].page+ page.locator("body[data-initial=ready]").wait_for(state="attached")+ for _ in range(250):+ if session.network.last_id >= 6:+ break+ page.wait_for_timeout(20)+ assert session.network.last_id >= 6+ last_id = _check_eviction(session.network)+ with page.expect_request_finished(lambda request: request.url.endswith("/api/post")):+ page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")+ _wait_saved(page, session.network, r"/api/post$")+ records = session.network.search()+ assert len(records) == 1 and records[0].id > last_id+ assert records[0].request_body == records[0].response.body == b"fresh"++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_browser_eviction_and_clear(session_type: Any, network_url: str) -> None:+ async with session_type(**_options(max_recorded_requests=2)) as session:+ await session.fetch(network_url + "/")+ page = session.page_pool.pages[0].page+ await page.locator("body[data-initial=ready]").wait_for(state="attached")+ for _ in range(250):+ if session.network.last_id >= 6:+ break+ await asyncio.sleep(0.02)+ assert session.network.last_id >= 6+ last_id = _check_eviction(session.network)+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/post")):+ await page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")+ await _wait_saved_async(page, session.network, r"/api/post$")+ records = session.network.search()+ assert len(records) == 1 and records[0].id > last_id+ assert records[0].request_body == records[0].response.body == b"fresh"+++PROXY_HTML = '<html><body><script>fetch("/api").then(response => response.json()).then(value => document.body.dataset.api = JSON.stringify(value));</script></body></html>'++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> None:+ def setup(page: Any) -> None:+ page.route(+ "**/*",+ lambda route: route.fulfill(+ content_type="application/json" if route.request.url.endswith("/api") else "text/html",+ body='{"temporary":true}' if route.request.url.endswith("/api") else PROXY_HTML,+ ),+ )++ def initial(page: Any) -> None:+ page.locator("body[data-api]").wait_for(state="attached")+ _wait_saved(page, session.network, r"/api$")++ with session_type(+ **_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]), page_setup=setup, page_action=initial)+ ) as session:+ session.fetch("https://network.test/")+ assert not session.page_pool.pages+ record = session.network.search(url_pattern=r"/api$")[0]+ assert record.status == 200 and record.resource_type == "fetch"+ assert record.response.json() == {"temporary": True}+ assert session.network.get(record.id).url == "https://network.test/api"+ assert record.response.json() == {"temporary": True}++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_browser_temporary_proxy_context(session_type: Any) -> None:+ async def setup(page: Any) -> None:+ await page.route(+ "**/*",+ lambda route: route.fulfill(+ content_type="application/json" if route.request.url.endswith("/api") else "text/html",+ body='{"temporary":true}' if route.request.url.endswith("/api") else PROXY_HTML,+ ),+ )++ async def initial(page: Any) -> None:+ await page.locator("body[data-api]").wait_for(state="attached")+ await _wait_saved_async(page, session.network, r"/api$")++ async with session_type(+ **_options(proxy_rotator=ProxyRotator(["http://127.0.0.1:1"]), page_setup=setup, page_action=initial)+ ) as session:+ await session.fetch("https://network.test/")+ assert not session.page_pool.pages+ record = session.network.search(url_pattern=r"/api$")[0]+ assert record.status == 200 and record.resource_type == "fetch"+ assert record.response.json() == {"temporary": True}+ assert session.network.get(record.id).url == "https://network.test/api"+ assert record.response.json() == {"temporary": True}+++def _check_saved_override(network: Any, network_url: str) -> None:+ records = network.search(url_pattern=r"/api/overridden$", method="POST", status=201)+ assert len(records) == 1+ record = records[0]+ assert record.url == network_url + "/api/overridden"+ assert record.request_body == record.response.body == b"overridden"+ assert record.request_headers["x-overridden"] == "yes"+ assert record.response.request_headers["x-overridden"] == "yes"+ assert not network.search(url_pattern=r"/api/original$")++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_browser_keeps_route_overrides_after_close(+ session_type: Any, network_url: str, monkeypatch: Any+) -> None:+ logger = Mock()+ monkeypatch.setattr("scrapling.engines.toolbelt.custom.log", logger)+ with session_type(**_options()) as session:+ session.fetch(network_url + "/")+ page = session.page_pool.pages[0].page+ page.route(+ "**/api/original",+ lambda route: route.continue_(+ url=network_url + "/api/overridden",+ method="POST",+ post_data="overridden",+ headers={**route.request.headers, "x-overridden": "yes"},+ ),+ )+ with page.expect_request_finished(lambda request: request.url.endswith("/api/overridden")):+ page.evaluate("fetch('/api/original').then(response => response.text())")+ _wait_saved(page, session.network, r"/api/overridden$")+ _check_saved_override(session.network, network_url)+ logger.info.assert_called_once()+ assert f"<GET {network_url}/>" in logger.info.call_args.args[0]++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_browser_keeps_route_overrides_after_close(+ session_type: Any, network_url: str, monkeypatch: Any+) -> None:+ logger = Mock()+ monkeypatch.setattr("scrapling.engines.toolbelt.custom.log", logger)+ async with session_type(**_options()) as session:+ await session.fetch(network_url + "/")+ page = session.page_pool.pages[0].page+ await page.route(+ "**/api/original",+ lambda route: route.continue_(+ url=network_url + "/api/overridden",+ method="POST",+ post_data="overridden",+ headers={**route.request.headers, "x-overridden": "yes"},+ ),+ )+ async with page.expect_request_finished(lambda request: request.url.endswith("/api/overridden")):+ await page.evaluate("fetch('/api/original').then(response => response.text())")+ await _wait_saved_async(page, session.network, r"/api/overridden$")+ _check_saved_override(session.network, network_url)+ logger.info.assert_called_once()+ assert f"<GET {network_url}/>" in logger.info.call_args.args[0]++[email protected]("session_type", [DynamicSession, StealthySession])[email protected]("target", ["page", "context", "session"])+def test_sync_network_close_during_unfinished_response(+ session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]+) -> None:+ path = f"/api/held/{session_type.__name__}/{target}"+ gate = network_gates[path] = _Gate()+ with session_type(**_options()) as session:+ session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ saved = _wait_saved(page, session.network, r"/blank$")[0]+ try:+ page.evaluate("fetch('/api/fail').catch(() => null)")+ with page.expect_response(network_url + path):+ page.evaluate("path => { fetch(path).then(response => response.text()).catch(() => null); }", path)+ assert gate.started.wait(5), "The server did not start the held response"+ assert not gate.release.is_set() and not gate.finished.is_set()+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")+ started = monotonic()+ {"page": session.close_pages, "context": session.context.close, "session": session.close}[target]()+ assert monotonic() - started < 5+ assert not gate.release.is_set() and not gate.finished.is_set()+ assert page.is_closed()+ assert session.network.get(saved.id).response.body == BLANK+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")+ assert bool(session.network._contexts) == (target == "page")+ finally:+ gate.release.set()+ assert gate.finished.wait(5), "The server did not finish the released response"+ assert not session.network._contexts+ assert session.network.search() == [saved]+ assert saved.response.body == BLANK++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("target", ["page", "context", "session"])+async def test_async_network_close_during_unfinished_response(+ session_type: Any, target: str, network_url: str, network_gates: dict[str, _Gate]+) -> None:+ path = f"/api/held/{session_type.__name__}/{target}"+ gate = network_gates[path] = _Gate()+ async with session_type(**_options()) as session:+ await session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ saved = (await _wait_saved_async(page, session.network, r"/blank$"))[0]+ try:+ await page.evaluate("fetch('/api/fail').catch(() => null)")+ async with page.expect_response(network_url + path):+ await page.evaluate(+ "path => { fetch(path).then(response => response.text()).catch(() => null); }", path+ )+ assert await asyncio.to_thread(gate.started.wait, 5), "The server did not start the held response"+ assert not gate.release.is_set() and not gate.finished.is_set()+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")+ close = {"page": session.close_pages, "context": session.context.close, "session": session.close}[target]+ await asyncio.wait_for(close(), timeout=5)+ assert not gate.release.is_set() and not gate.finished.is_set()+ assert page.is_closed()+ assert session.network.get(saved.id).response.body == BLANK+ assert not session.network.search(url_pattern=r"/api/(?:held|fail)")+ assert bool(session.network._contexts) == (target == "page")+ finally:+ gate.release.set()+ assert await asyncio.to_thread(gate.finished.wait, 5), "The server did not finish the released response"+ assert not session.network._contexts+ assert session.network.search() == [saved]+ assert saved.response.body == BLANK+++def _check_burst(network: Any, cursor: int, path: str, expected_paths: list[str]) -> int:+ newer = network.search(after_id=cursor, limit=1)+ assert len(newer) == 1 and urlsplit(newer[0].url).path == path+ assert newer[0].id == cursor + 1+ records = network.search(limit=2)+ records += network.search(after_id=records[-1].id, limit=2)+ assert [urlsplit(record.url).path for record in records] == expected_paths[-3:]+ assert all(record.response.json() == {"ok": True} for record in records)+ assert network.dropped_count == max(0, len(expected_paths) - 3)+ return newer[0].id++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_burst_completion_order_and_pagination(+ session_type: Any, network_url: str, network_gates: dict[str, _Gate]+) -> None:+ paths = [f"/api/burst/{session_type.__name__}/{index}" for index in range(6)]+ gates = {path: _Gate() for path in paths}+ network_gates.update(gates)+ with session_type(**_options(max_recorded_requests=3)) as session:+ session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ _wait_saved(page, session.network, r"/blank$")+ session.network.clear()+ cursor = session.network.last_id+ completed: list[str] = []+ try:+ page.evaluate(+ "paths => { for (const path of paths) fetch(path).then(response => response.text()).catch(() => null); }",+ paths,+ )+ assert all(gate.started.wait(5) for gate in gates.values()), "The request burst did not reach the server"+ assert session.network.search() == []+ for index in (5, 2, 4, 1, 3, 0):+ path = paths[index]+ gates[path].release.set()+ _wait_saved(page, session.network, path + "$", after_id=cursor)+ completed.append(path)+ cursor = _check_burst(session.network, cursor, path, completed)+ assert session.network.get(cursor - 3) is None+ finally:+ for gate in gates.values():+ gate.release.set()+ assert all(gate.finished.wait(5) for gate in gates.values())+ assert not session.network._contexts+ records = session.network.search()+ assert [urlsplit(record.url).path for record in records] == completed[-3:]+ assert all(record.response.json() == {"ok": True} for record in records)++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_burst_completion_order_and_pagination(+ session_type: Any, network_url: str, network_gates: dict[str, _Gate]+) -> None:+ paths = [f"/api/burst/{session_type.__name__}/{index}" for index in range(6)]+ gates = {path: _Gate() for path in paths}+ network_gates.update(gates)+ async with session_type(**_options(max_recorded_requests=3)) as session:+ await session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ await _wait_saved_async(page, session.network, r"/blank$")+ session.network.clear()+ cursor = session.network.last_id+ completed: list[str] = []+ try:+ await page.evaluate(+ "paths => { for (const path of paths) fetch(path).then(response => response.text()).catch(() => null); }",+ paths,+ )+ for gate in gates.values():+ assert await asyncio.to_thread(gate.started.wait, 5), "The request burst did not reach the server"+ assert session.network.search() == []+ for index in (5, 2, 4, 1, 3, 0):+ path = paths[index]+ gates[path].release.set()+ await _wait_saved_async(page, session.network, path + "$", after_id=cursor)+ completed.append(path)+ cursor = _check_burst(session.network, cursor, path, completed)+ assert session.network.get(cursor - 3) is None+ finally:+ for gate in gates.values():+ gate.release.set()+ for gate in gates.values():+ assert await asyncio.to_thread(gate.finished.wait, 5)+ assert not session.network._contexts+ records = session.network.search()+ assert [urlsplit(record.url).path for record in records] == completed[-3:]+ assert all(record.response.json() == {"ok": True} for record in records)+++def _check_wire_bodies(+ network: Any, wire: dict[str, list[int]], native_bodies: dict[str, bytes], native_texts: dict[str, str]+) -> None:+ records = {urlsplit(record.url).path: record for record in network.search(url_pattern=r"/wire/")}+ assert set(records) == set(WIRE_PATHS)+ for path in ("/wire/chunked", "/wire/gzip"):+ assert records[path].response.meta.get("body_note") == "too large; not saved."+ assert records[path].response.body == b""+ assert "content-length" not in records["/wire/chunked"].response_headers+ assert records["/wire/chunked"].response_headers["transfer-encoding"] == "chunked"+ assert int(records["/wire/gzip"].response_headers["content-length"]) < 1024 * 1024+ assert records["/wire/gzip"].response_headers["content-encoding"] == "gzip"+ assert bytes(wire["/wire/binary"]) == native_bodies["/wire/binary"] == BINARY+ for path in ("/wire/binary", "/wire/raw-charset"):+ assert records[path].response.body == b""+ assert records[path].response.meta.get("body_note") == "Non-text body; not saved."+ details = _request_details(records[path], "response_body")+ assert details.data is None and details.note == "Non-text body; not saved."+ for path in CHARSET_PATHS:+ assert bytes(wire[path]) == CHARSET_BODY+ assert records[path].response_headers["content-type"].lower().endswith(("iso-8859-15", '"iso-8859-15"'))+ for path, text in native_texts.items():+ assert records[path].response.body == native_bodies[path]+ assert "body_note" not in records[path].response.meta+ assert records[path].response.encoding == "utf-8"+ assert native_bodies[path].decode("utf-8") == text+ assert records[path].response.get_all_text().strip() == text+ details = _request_details(records[path], "response_body")+ assert details.data == text and details.request_id == records[path].id+ assert details.part == "response_body" and details.note is None+ assert native_texts["/wire/charset"] == "caf\u00e9 \u20ac"+ assert native_bodies["/wire/charset"] == "caf\u00e9 \u20ac".encode("utf-8")+ assert native_bodies["/wire/raw-charset"] == CHARSET_BODY+ assert records["/wire/head"].method == "HEAD" and records["/wire/head"].status == 200+ assert records["/wire/empty"].status == 204+ for path in ("/wire/head", "/wire/empty"):+ assert records[path].response.body == b"" and "body_note" not in records[path].response.meta++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:+ with session_type(**_options()) as session:+ session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ native: dict[str, Any] = {}+ page.on("response", lambda response: native.update({urlsplit(response.url).path: response}))+ wire = page.evaluate(WIRE_FETCH, WIRE_PATHS)+ _wait_saved(page, session.network, r"/wire/", len(WIRE_PATHS))+ native_bodies = {path: native[path].body() for path in wire}+ native_texts = {path: native[path].text() for path in CHARSET_PATHS[:2]}+ with pytest.raises(UnicodeDecodeError):+ native["/wire/raw-charset"].text()+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)+ assert not session.network._contexts+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_wire_bodies_remain_readable_after_close(session_type: Any, network_url: str) -> None:+ async with session_type(**_options()) as session:+ await session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ native: dict[str, Any] = {}+ page.on("response", lambda response: native.update({urlsplit(response.url).path: response}))+ wire = await page.evaluate(WIRE_FETCH, WIRE_PATHS)+ await _wait_saved_async(page, session.network, r"/wire/", len(WIRE_PATHS))+ native_bodies = {path: await native[path].body() for path in wire}+ native_texts = {path: await native[path].text() for path in CHARSET_PATHS[:2]}+ with pytest.raises(UnicodeDecodeError):+ await native["/wire/raw-charset"].text()+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)+ assert not session.network._contexts+ _check_wire_bodies(session.network, wire, native_bodies, native_texts)+++def _check_mime_bodies(network: Any, wire: dict[str, list[int]], reads: list[str]) -> None:+ records = {urlsplit(record.url).path: record for record in network.search(url_pattern=r"/mime/")}+ assert set(records) == set(wire) == set(MIME_RESPONSES)+ assert sorted(reads) == sorted(TEXT_RESPONSES)+ for path, (content_type, body) in MIME_RESPONSES.items():+ record = records[path]+ assert bytes(wire[path]) == body+ assert record.status == 200 and record.method == "GET" and record.resource_type == "fetch"+ assert record.response_headers.get("content-type") == content_type+ assert record.response_headers["x-recorded"] == "response"+ assert int(record.response_headers["content-length"]) == len(body)+ if path in TEXT_RESPONSES:+ assert record.response.body == body and "body_note" not in record.response.meta+ else:+ assert record.response.body == b""+ assert record.response.meta.get("body_note") == "Non-text body; not saved."+ for path in ("/mime/json", "/mime/vendor-json"):+ assert records[path].response.json() == {"saved": True}+++ ("session_type", "response_type"), [(DynamicSession, PlaywrightResponse), (StealthySession, PatchrightResponse)]+)+def test_sync_network_reads_only_text_bodies(+ session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ reads: list[str] = []+ native_body = response_type.body++ def read_body(response: Any) -> bytes:+ path = urlsplit(response.url).path+ if path in MIME_RESPONSES:+ reads.append(path)+ return native_body(response)++ monkeypatch.setattr(response_type, "body", read_body)+ with session_type(**_options()) as session:+ session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ wire = page.evaluate(MIME_FETCH, list(MIME_RESPONSES))+ _wait_saved(page, session.network, r"/mime/", len(MIME_RESPONSES))+ _check_mime_bodies(session.network, wire, reads)+ assert not session.network._contexts+ _check_mime_bodies(session.network, wire, reads)+++ ("session_type", "response_type"),+ [(AsyncDynamicSession, AsyncPlaywrightResponse), (AsyncStealthySession, AsyncPatchrightResponse)],+)+async def test_async_network_reads_only_text_bodies(+ session_type: Any, response_type: Any, network_url: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ reads: list[str] = []+ native_body = response_type.body++ async def read_body(response: Any) -> bytes:+ path = urlsplit(response.url).path+ if path in MIME_RESPONSES:+ reads.append(path)+ return await native_body(response)++ monkeypatch.setattr(response_type, "body", read_body)+ async with session_type(**_options()) as session:+ await session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ wire = await page.evaluate(MIME_FETCH, list(MIME_RESPONSES))+ await _wait_saved_async(page, session.network, r"/mime/", len(MIME_RESPONSES))+ _check_mime_bodies(session.network, wire, reads)+ assert not session.network._contexts+ _check_mime_bodies(session.network, wire, reads)diff --git a/tests/fetchers/test_network_cleanup.py b/tests/fetchers/test_network_cleanup.pynew file mode 100644index 0000000..dc67a75--- /dev/null+++ b/tests/fetchers/test_network_cleanup.py@@ -0,0 +1,83 @@+import asyncio+from unittest.mock import AsyncMock, Mock++import pytest++from scrapling.engines._browsers._base import AsyncSession, SyncSession+from tests.fetchers.test_network_history import request+++def _session(asynchronous=False):+ session = AsyncSession(record_requests=True) if asynchronous else SyncSession(record_requests=True)+ context, browser, playwright = Mock(), Mock(), Mock()+ if asynchronous:+ context.close, browser.close, playwright.stop = AsyncMock(), AsyncMock(), AsyncMock()+ session.context, session.browser, session.playwright = context, browser, playwright+ session._is_alive = True+ session.network._attach(context)+ capture = dict(call.args for call in context.on.call_args_list)["requestfinished"]+ return session, context, browser, playwright, capture+++def _closed(session, context, browser, playwright):+ assert not session._is_alive and not session.network._contexts+ assert session.context is session.browser is session.playwright is None+ context.close.assert_called_once_with()+ browser.close.assert_called_once_with()+ playwright.stop.assert_called_once_with()+ assert {call.args[0] for call in context.remove_listener.call_args_list} == {"requestfinished", "close"}+++def test_sync_session_close_during_capture_does_not_restore_late_response():+ session, context, browser, playwright, capture = _session()+ capture(request(body=b"saved"))+ saved = session.network.get(1)++ def close_before_body_returns():+ session.close()+ return b"late"++ native = request()+ native.existing_response.body.side_effect = close_before_body_returns+ capture(native)+ session.close()+ _closed(session, context, browser, playwright)+ assert session.network.search() == [saved] and saved.response.body == b"saved"+ assert session.network.last_id == 1++[email protected]("read_error", [False, True])+async def test_async_session_close_does_not_wait_for_running_capture(read_error):+ session, context, browser, playwright, capture = _session(asynchronous=True)+ await capture(request(body=b"saved", asynchronous=True))+ saved = session.network.get(1)+ reading, release = asyncio.Event(), asyncio.Event()++ async def held_body():+ reading.set()+ await release.wait()+ if read_error:+ raise RuntimeError("Browser closed while reading")+ return b"late"++ native = request(asynchronous=True)+ native.existing_response.body.side_effect = held_body+ task = asyncio.create_task(capture(native))+ try:+ await asyncio.wait_for(reading.wait(), timeout=2)+ await asyncio.wait_for(session.close(), timeout=2)+ assert not task.done() and not release.is_set()+ _closed(session, context, browser, playwright)+ context.close.assert_awaited_once_with()+ browser.close.assert_awaited_once_with()+ playwright.stop.assert_awaited_once_with()+ release.set()+ await asyncio.wait_for(task, timeout=2)+ await session.close()+ assert session.network.search() == [saved] and saved.response.body == b"saved"+ assert session.network.last_id == 1+ finally:+ release.set()+ await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=2)+ await session.close()diff --git a/tests/fetchers/test_network_config.py b/tests/fetchers/test_network_config.pynew file mode 100644index 0000000..585afe2--- /dev/null+++ b/tests/fetchers/test_network_config.py@@ -0,0 +1,32 @@+import pytest++from scrapling.core._types import Any+from scrapling.engines._browsers._validators import PlaywrightConfig, StealthConfig, validate+from scrapling.fetchers import DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession++[email protected]("model", [PlaywrightConfig, StealthConfig])+def test_network_config_defaults(model: type[PlaywrightConfig]) -> None:+ config = validate({}, model)+ assert config.record_requests is False+ assert config.max_recorded_requests == 1000++[email protected]("model", [PlaywrightConfig, StealthConfig])+ "values", [{"record_requests": "yes"}, {"max_recorded_requests": 0}, {"max_recorded_requests": -1}]+)+def test_network_config_invalid(model: type[PlaywrightConfig], values: dict[str, Any]) -> None:+ with pytest.raises(TypeError):+ validate(values, model)++[email protected]("session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession])[email protected]("enabled", [False, True])+def test_sessions_configure_one_network_history(session_type: Any, enabled: bool) -> None:+ session = session_type(record_requests=enabled, max_recorded_requests=7)+ assert session.network.enabled is enabled+ assert session._config.max_recorded_requests == 7+ assert session.network.search() == []+ assert session.network.get(1) is None+ assert session.network.last_id == 0diff --git a/tests/fetchers/test_network_history.py b/tests/fetchers/test_network_history.pynew file mode 100644index 0000000..1743b0b--- /dev/null+++ b/tests/fetchers/test_network_history.py@@ -0,0 +1,559 @@+import asyncio+import gc+from inspect import isawaitable+from weakref import ref+from unittest.mock import AsyncMock, Mock, PropertyMock++import pytest++from scrapling.core._types import Any, Optional+from scrapling.engines._browsers import _network+from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest+from scrapling.engines.toolbelt.custom import Response+++class Context:+ def __init__(self):+ self.listeners: dict[str, list[Any]] = {}++ def on(self, event, listener):+ self.listeners.setdefault(event, []).append(listener)++ def remove_listener(self, event, listener):+ self.listeners[event].remove(listener)++ def emit(self, event, value=None):+ tasks = []+ for listener in self.listeners.get(event, []).copy():+ result = listener(value)+ if isawaitable(result):+ tasks.append(asyncio.ensure_future(result))+ return tasks+++def request(+ url="https://example.test/api",+ method="GET",+ resource_type="fetch",+ status: Optional[int] = 200,+ body=b'{"ok":true}',+ asynchronous=False,+):+ make = AsyncMock if asynchronous else Mock+ native = Mock(url=url, method=method, resource_type=resource_type)+ native.headers = {"cookie": "session=1"}+ native.all_headers = make(return_value=native.headers.copy())+ native.post_data_buffer = None+ native.existing_response = None+ if status is not None:+ response = Mock(url=url, status=status, status_text="OK", request=native)+ response.headers = {"content-type": "application/json", "content-length": str(len(body))}+ response.all_headers = make(return_value=response.headers.copy())+ response.body = make(return_value=body)+ native.existing_response = response+ return native+++def recorder(max_requests=1000, asynchronous=False):+ network = NetworkRecorder(enabled=True, max_requests=max_requests, asynchronous=asynchronous)+ context = Context()+ network._attach(context)+ return network, context+++async def finish(context, native):+ await asyncio.gather(*context.emit("requestfinished", native))++[email protected]("asynchronous", [False, True])+async def test_only_completed_responses_are_saved(asynchronous):+ network, context = recorder(asynchronous=asynchronous)+ native = request(asynchronous=asynchronous)+ assert set(context.listeners) == {"requestfinished", "close"}+ context.emit("request", native)+ context.emit("response", native.existing_response)+ context.emit("requestfailed", request(status=None, asynchronous=asynchronous))+ assert network.search() == [] and network.last_id == 0+ native.all_headers.assert_not_called()+ native.existing_response.body.assert_not_called()+ await finish(context, request(status=None, asynchronous=asynchronous))+ assert network.search() == []+ await finish(context, native)+ record = network.get(1)+ assert isinstance(record, NetworkRequest) and not hasattr(record, "request")+ assert not hasattr(record, "state") and not hasattr(record, "failure")+ assert record.id == 1 and record.url == native.url and record.method == "GET"+ assert record.resource_type == "fetch" and record.status == 200+ assert record.response.json() == {"ok": True}+ assert "body_note" not in record.response.meta++[email protected]("asynchronous", [False, True])+async def test_saved_response_has_final_request_data_and_survives_close(asynchronous):+ network, context = recorder(asynchronous=asynchronous)+ native = request(asynchronous=asynchronous)+ original_url = native.url+ context.emit("request", native)+ native.url, native.method = "https://example.test/overridden", "POST"+ native.headers = {"x-overridden": "yes", "content-type": "text/plain"}+ native.post_data_buffer = b"overridden"+ native.all_headers.return_value = native.headers.copy()+ native.existing_response.url = native.url+ await finish(context, native)+ record = network.get(1)+ assert record is not None+ assert record.url == native.url and record.response.url == native.url and record.method == "POST"+ assert record.request_body == b"overridden"+ assert record.request_headers == record.response.request_headers == native.headers+ assert record.response_headers == record.response.headers == native.existing_response.headers+ assert network.search(url_pattern=original_url) == [] and network.search(method="POST") == [record]+ native.headers["x-overridden"] = "changed"+ native.existing_response.headers["content-type"] = "changed"+ assert record.request_headers["x-overridden"] == "yes"+ assert record.response_headers["content-type"] == "application/json"+ native.all_headers.assert_called_once_with()+ native.existing_response.all_headers.assert_called_once_with()+ native.existing_response.body.assert_called_once_with()+ context.emit("close")+ network._detach()+ native.all_headers.side_effect = RuntimeError("Browser closed")+ native.existing_response.all_headers.side_effect = RuntimeError("Browser closed")+ native.existing_response.body.side_effect = RuntimeError("Browser closed")+ assert record.response.json() == {"ok": True}+ assert network.get(1) is record and network.search() == [record]+ native.existing_response.body.assert_called_once_with()++[email protected]("asynchronous", [False, True])[email protected]("outcome", ["saved", "oversized", "unreadable", "invalid_charset", "non_text"])+async def test_recording_skips_fetch_log_without_muting_normal_responses(monkeypatch, asynchronous, outcome):+ logger = Mock()+ monkeypatch.setattr("scrapling.engines.toolbelt.custom.log", logger)+ network, context = recorder(asynchronous=asynchronous)+ native = request(asynchronous=asynchronous)+ if outcome == "oversized":+ monkeypatch.setattr(_network, "MAX_BODY_BYTES", 4)+ elif outcome == "unreadable":+ native.existing_response.body.side_effect = RuntimeError("Body missing")+ elif outcome == "invalid_charset":+ native.existing_response.headers["content-type"] = "application/json; charset=bogus"+ elif outcome == "non_text":+ native.existing_response.headers["content-type"] = "application/octet-stream"+ await finish(context, native)+ record = network.get(1)+ assert record is not None+ assert (+ record.response.meta+ == {+ "saved": {},+ "oversized": {"body_note": "too large; not saved."},+ "unreadable": {"body_note": "Body missing"},+ "invalid_charset": {},+ "non_text": {"body_note": "Non-text body; not saved."},+ }[outcome]+ )+ logger.info.assert_not_called()+ response = Response(+ url="https://example.test/page",+ content=b"page",+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={},+ )+ assert response.body == b"page"+ logger.info.assert_called_once_with("Fetched (200) <GET https://example.test/page> (referer: None)")++[email protected]("asynchronous", [False, True])+async def test_oversized_response_keeps_headers_without_reading_body(monkeypatch, asynchronous):+ monkeypatch.setattr(_network, "MAX_BODY_BYTES", 4)+ network, context = recorder(asynchronous=asynchronous)+ native = request(body=b"large", asynchronous=asynchronous)+ await finish(context, native)+ record = network.get(1)+ assert record is not None+ assert record.status == 200 and record.response.meta == {"body_note": "too large; not saved."}+ assert record.response.body == b"" and record.response.headers["content-length"] == "5"+ native.existing_response.body.assert_not_called()++[email protected]("asynchronous", [False, True])[email protected]("message", ["Body was removed", ""])+async def test_unreadable_response_keeps_headers_and_reports_unavailable(asynchronous, message):+ network, context = recorder(asynchronous=asynchronous)+ native = request(asynchronous=asynchronous)+ native.existing_response.body.side_effect = RuntimeError(message)+ await finish(context, native)+ record = network.get(1)+ assert record is not None+ assert record.status == 200+ assert record.response.meta == {"body_note": message or "could not be saved."}+ assert record.response.body == b"" and record.response.headers == native.existing_response.headers++[email protected]("asynchronous", [False, True])+async def test_non_text_records_keep_metadata_without_using_body_budget(monkeypatch, asynchronous):+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 8)+ network, context = recorder(asynchronous=asynchronous)+ await finish(context, request(body=b"1234", asynchronous=asynchronous))+ for request_id, content_type in enumerate(("application/octet-stream", "application/x-custom", None), start=2):+ native = request(+ url=f"https://example.test/binary/{request_id}",+ method="POST",+ resource_type="image",+ status=206,+ body=b"\x00\xffbinary",+ asynchronous=asynchronous,+ )+ native.post_data_buffer = b"submitted"+ response = native.existing_response+ response.headers = {"x-saved": "yes"}+ if content_type is not None:+ response.headers["content-type"] = content_type+ response.all_headers.return_value = response.headers.copy()+ response.body.side_effect = AssertionError("Non-text body must not be read")+ await finish(context, native)+ record = network.get(request_id)+ assert record is not None+ assert record.url == native.url and record.method == "POST" and record.status == 206+ assert record.resource_type == "image" and record.request_body == b"submitted"+ assert record.request_headers == native.headers and record.response_headers == response.headers+ assert record.response.body == b"" and record.response.meta == {"body_note": "Non-text body; not saved."}+ response.body.assert_not_called()+ await finish(context, request(body=b"5678", asynchronous=asynchronous))+ assert [record.id for record in network.search()] == [1, 2, 3, 4, 5]+ assert [record.response.body for record in network.search()] == [b"1234", b"", b"", b"", b"5678"]+ assert network.dropped_count == 0+ await finish(context, request(body=b"9", asynchronous=asynchronous))+ assert [record.id for record in network.search()] == [2, 3, 4, 5, 6]+ assert network.get(1) is None and network.dropped_count == 1++[email protected]("asynchronous", [False, True])+async def test_total_body_budget_evicts_oldest_records_and_clear_releases_budget(monkeypatch, asynchronous):+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 7)+ network, context = recorder(asynchronous=asynchronous)+ for body in (b"one", b"two", b"three"):+ await finish(context, request(body=body, asynchronous=asynchronous))+ assert [record.id for record in network.search()] == [3]+ record = network.get(3)+ assert record is not None+ assert network.dropped_count == 2 and record.response.body == b"three"+ network.clear()+ assert network.dropped_count == 0 and network.last_id == 3 and not network.search()+ for _ in range(2):+ await finish(context, request(body=b"new", asynchronous=asynchronous))+ assert [record.id for record in network.search()] == [4, 5] and network.dropped_count == 0++[email protected]("asynchronous", [False, True])+async def test_count_eviction_releases_saved_body_budget(monkeypatch, asynchronous):+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 5)+ network, context = recorder(max_requests=1, asynchronous=asynchronous)+ for _ in range(3):+ await finish(context, request(body=b"new", asynchronous=asynchronous))+ assert [record.id for record in network.search()] == [3]+ record = network.get(3)+ assert record is not None+ assert record.response.body == b"new" and network.dropped_count == 2+ network.clear()+ await finish(context, request(body=b"new", asynchronous=asynchronous))+ assert [record.id for record in network.search()] == [4] and network.dropped_count == 0++[email protected]("change", ["clear", "detach", "reattach"])[email protected]("started", [False, True])+async def test_late_callback_cannot_publish_after_clear_or_detach(change, started):+ network, context = recorder(asynchronous=True)+ reading, release = asyncio.Event(), asyncio.Event()++ async def body():+ reading.set()+ await release.wait()+ return b"old"++ native = request(asynchronous=True)+ native.existing_response.body.side_effect = body+ tasks = context.emit("requestfinished", native)+ try:+ if started:+ await asyncio.wait_for(reading.wait(), timeout=2)+ if change == "clear":+ network.clear()+ else:+ network._detach(context)+ if change == "reattach":+ network._attach(context)+ release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=2)+ assert network.search() == [] and network.last_id == 0+ if change != "detach":+ await finish(context, request(body=b"new", asynchronous=True))+ record = network.get(1)+ assert record is not None+ assert record.response.body == b"new"+ finally:+ release.set()+ await asyncio.gather(*tasks)+ network._detach()+++async def test_records_are_published_in_save_order_without_waiting_for_other_bodies():+ network, context = recorder(asynchronous=True)+ reading, release = asyncio.Event(), asyncio.Event()++ async def body():+ reading.set()+ await release.wait()+ return b"first"++ first = request(asynchronous=True)+ first.existing_response.body.side_effect = body+ tasks = context.emit("requestfinished", first)+ try:+ await asyncio.wait_for(reading.wait(), timeout=2)+ assert network.search() == [] and network.get(1) is None+ await finish(context, request(body=b"second", asynchronous=True))+ record = network.get(1)+ assert record is not None+ assert record.response.body == b"second"+ release.set()+ await asyncio.wait_for(asyncio.gather(*tasks), timeout=2)+ assert [record.response.body for record in network.search()] == [b"second", b"first"]+ finally:+ release.set()+ await asyncio.gather(*tasks)+++def test_context_lifecycle_keeps_history_and_removes_only_owned_listeners():+ network, first = recorder()+ second, external = Context(), Mock()+ first.on("requestfinished", external)+ network._attach(first)+ network._attach(second)+ assert len(first.listeners["requestfinished"]) == 2 and len(first.listeners["close"]) == 1+ first.emit("requestfinished", request(status=201))+ second.emit("requestfinished", request())+ network._detach(first)+ first.emit("requestfinished", request())+ second.emit("requestfinished", request())+ assert network.last_id == 3+ assert first.listeners["requestfinished"] == [external] and not first.listeners["close"]+ second.emit("close")+ assert not any(second.listeners.values()) and not network._contexts+ record = network.get(1)+ assert record is not None+ assert record.status == 201+ network._attach(first)+ first.emit("requestfinished", request())+ network._detach()+ network._detach()+ assert network.enabled and len(network.search()) == 4+ assert first.listeners["requestfinished"] == [external] and not first.listeners["close"]+ external.assert_called()+++def test_search_filters_include_completed_http_errors():+ network, context = recorder()+ for native in (+ request(url="https://example.test/page", resource_type="document"),+ request(url="https://example.test/api", method="POST", status=503),+ request(url="https://example.test/missing", resource_type="document", status=404),+ request(url="https://example.test/api", resource_type="xhr"),+ ):+ context.emit("requestfinished", native)+ assert [record.id for record in network.search(include_static=False)] == [2, 3, 4]+ assert [record.id for record in network.search(after_id=2, limit=1)] == [3]+ assert [record.id for record in network.search(url_pattern="/api$", method="post", status=503)] == [2]+ assert [record.id for record in network.search(resource_type="xhr")] == [4]+ assert network.get(100) is None and network.search(url_pattern="absent") == []++[email protected]("options", [{"limit": 0}, {"after_id": -1}, {"url_pattern": "["}])+def test_invalid_filters_raise_value_error(options):+ with pytest.raises(ValueError):+ NetworkRecorder().search(**options)+++def test_disabled_recorder_never_attaches():+ network, context = NetworkRecorder(), Mock()+ network._attach(context)+ network._detach(context)+ network._detach()+ assert not context.mock_calls and not network.search() and network.last_id == 0+ with pytest.raises(ValueError, match="greater than zero"):+ NetworkRecorder(max_requests=0)+++def test_partial_listener_setup_failure_does_not_leave_capture_attached():+ network, context = NetworkRecorder(enabled=True), Mock()+ context.on.side_effect = [None, RuntimeError("Context closed")]+ network._attach(context)+ assert not network._contexts+ assert context.remove_listener.call_count == 2++[email protected]("asynchronous", [False, True])[email protected]("outcome", ["finished", "body_error", "headers_error", "conversion_error", "unfinished"])+async def test_native_handles_are_not_kept_in_history(asynchronous, outcome):+ network, context = recorder(asynchronous=asynchronous)+ native = request(asynchronous=asynchronous)+ native_ref, response_ref = ref(native), ref(native.existing_response)+ if outcome == "body_error":+ native.existing_response.body.side_effect = RuntimeError("Body missing")+ elif outcome == "headers_error":+ native.existing_response.all_headers.side_effect = RuntimeError("Headers missing")+ elif outcome == "conversion_error":+ type(native.existing_response).status_text = PropertyMock(side_effect=RuntimeError("Response missing"))+ if outcome != "unfinished":+ await finish(context, native)+ else:+ context.emit("request", native)+ if outcome in {"unfinished", "conversion_error"}:+ assert network.search() == []+ else:+ record = network.get(1)+ assert record is not None+ assert record.status == 200+ assert record.response.meta.get("body_note") == ("Body missing" if outcome == "body_error" else None)+ del native+ gc.collect()+ assert native_ref() is None and response_ref() is None++[email protected]("change", ["clear", "detach", "reattach"])+def test_sync_conversion_cannot_restore_history_after_reentrant_reset(change):+ network, context = recorder()+ native = request(body=b"old")++ def body():+ assert network.search() == [] and network.get(1) is None+ if change == "clear":+ network.clear()+ else:+ network._detach(context)+ if change == "reattach":+ network._attach(context)+ if change != "detach":+ context.emit("requestfinished", request(body=b"new"))+ return b"old"++ native.existing_response.body.side_effect = body+ context.emit("requestfinished", native)+ expected = [] if change == "detach" else [b"new"]+ assert [record.response.body for record in network.search()] == expected+ assert network.last_id == len(expected)+++async def test_cancelled_conversion_publishes_nothing_and_later_requests_still_save():+ network, context = recorder(asynchronous=True)+ reading, release = asyncio.Event(), asyncio.Event()++ async def body():+ reading.set()+ await release.wait()+ return b"cancelled"++ native = request(asynchronous=True)+ native.existing_response.body.side_effect = body+ tasks = context.emit("requestfinished", native)+ try:+ await asyncio.wait_for(reading.wait(), timeout=2)+ tasks[0].cancel()+ with pytest.raises(asyncio.CancelledError):+ await tasks[0]+ assert network.search() == [] and network.last_id == 0+ await finish(context, request(body=b"saved", asynchronous=True))+ record = network.get(1)+ assert record is not None+ assert record.response.body == b"saved"+ finally:+ release.set()+ await asyncio.gather(*tasks, return_exceptions=True)+ network._detach()++[email protected]("asynchronous", [False, True])[email protected]("budget", ["count", "body", "both"])+async def test_exact_storage_boundaries_keep_records_until_exceeded(monkeypatch, asynchronous, budget):+ monkeypatch.setattr(_network, "MAX_BODY_BYTES", 4)+ monkeypatch.setattr(_network, "MAX_TOTAL_BODY_BYTES", 100 if budget == "count" else 8)+ network, context = recorder(max_requests=3 if budget == "body" else 2, asynchronous=asynchronous)+ for body in (b"1111", b"2222"):+ await finish(context, request(body=body, asynchronous=asynchronous))+ assert [record.response.body for record in network.search()] == [b"1111", b"2222"]+ assert network.dropped_count == 0+ await finish(context, request(body=b"3", asynchronous=asynchronous))+ assert [record.response.body for record in network.search()] == [b"2222", b"3"]+ assert network.get(1) is None and network.dropped_count == 1++[email protected]("asynchronous", [False, True])[email protected]("attribute", ["resource_type", "post_data_buffer"])+async def test_unreadable_request_metadata_is_omitted_and_native_handles_are_released(asynchronous, attribute):+ network, context = recorder(asynchronous=asynchronous)+ native = request(asynchronous=asynchronous)+ native_ref, response_ref = ref(native), ref(native.existing_response)+ setattr(type(native), attribute, PropertyMock(side_effect=RuntimeError("Request metadata missing")))+ await finish(context, native)+ assert network.search() == []+ del native+ gc.collect()+ assert native_ref() is None and response_ref() is None+ await finish(context, request(body=b"next", asynchronous=asynchronous))+ assert [record.response.body for record in network.search()] == [b"next"]++[email protected]("asynchronous", [False, True])+async def test_detach_ignores_removed_listener_errors_and_rejects_stale_callbacks(asynchronous):+ network, context = recorder(asynchronous=asynchronous)+ context.remove_listener = Mock(side_effect=RuntimeError("Context already closed"))+ network._detach(context)+ await finish(context, request(asynchronous=asynchronous))+ assert network.search() == [] and network.last_id == 0+ context.remove_listener.assert_any_call("requestfinished", context.listeners["requestfinished"][0])+ context.remove_listener.assert_any_call("close", context.listeners["close"][0])++[email protected]("asynchronous", [False, True])+ "body,expected,encoding",+ [+ ("café €".encode(), "café €", "utf-8"),+ (b"caf\xe9 \xa4", "café €", "iso-8859-15"),+ (b"\xc3\xa9", "é", "utf-8"),+ ],+)+async def test_saved_native_bytes_and_text_encoding_remain_valid_after_detach(asynchronous, body, expected, encoding):+ network, context = recorder(asynchronous=asynchronous)+ native = request(body=body, asynchronous=asynchronous)+ content_type = "text/plain; charset=iso-8859-15"+ native.existing_response.headers["content-type"] = content_type+ native.existing_response.all_headers.return_value = native.existing_response.headers.copy()+ await finish(context, native)+ network._detach()+ record = network.get(1)+ assert record is not None+ assert record.response.body == body+ assert record.response_headers["content-type"] == content_type+ assert record.response.encoding == encoding+ assert record.response.get_all_text().strip() == expected+ assert "body_note" not in record.response.metadiff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.pyindex b422ead..152d525 100644--- a/tests/fetchers/test_response_handling.py+++ b/tests/fetchers/test_response_handling.py@@ -18,6 +18,8 @@ def make_sync_playwright_response():response.headers = {"content-type": LATIN9_CONTENT_TYPE}response.all_headers = Mock(return_value={"content-type": LATIN9_CONTENT_TYPE})response.body = Mock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))+ response.request.method = "GET"+ response.request.headers = {}response.request.all_headers = Mock(return_value={})response.request.redirected_from = Nonereturn response@@ -39,6 +41,8 @@ def make_async_playwright_response():response.headers = {"content-type": LATIN9_CONTENT_TYPE}response.all_headers = AsyncMock(return_value={"content-type": LATIN9_CONTENT_TYPE})response.body = AsyncMock(return_value=HTML_WITH_ACCENTED_TEXT.encode("iso-8859-15"))+ response.request.method = "GET"+ response.request.headers = {}response.request.all_headers = AsyncMock(return_value={})response.request.redirected_from = Nonereturn response@@ -70,10 +74,7 @@ class TestResponseFactory:mock_curl_response.request.method = "GET"mock_curl_response.history = []- response = ResponseFactory.from_http_request(- mock_curl_response,- {"adaptive": False}- )+ response = ResponseFactory.from_http_request(mock_curl_response, {"adaptive": False})assert response.status == 200assert response.url == "https://example.com"@@ -96,9 +97,15 @@ class TestResponseFactory:assert "Bürger" in response.html_contentassert "BÃ" not in response.html_content- def test_playwright_raw_response_keeps_header_encoding(self):+ @pytest.mark.parametrize(+ "content_type",+ [LATIN9_CONTENT_TYPE, 'text/html; CHARSET = "iso-8859-15"', "text/html; charset = 'iso-8859-15'"],+ )+ def test_playwright_raw_response_keeps_header_encoding(self, content_type):"""Raw response bytes should still use the charset from Content-Type."""mock_response = make_sync_playwright_response()+ mock_response.headers = {"content-type": content_type}+ mock_response.all_headers.return_value = mock_response.headersresponse = ResponseFactory.from_playwright_response(None,@@ -130,9 +137,15 @@ class TestResponseFactory:assert "BÃ" not in response.html_content@pytest.mark.asyncio- async def test_async_playwright_raw_response_keeps_header_encoding(self):+ @pytest.mark.parametrize(+ "content_type",+ [LATIN9_CONTENT_TYPE, 'text/html; CHARSET = "iso-8859-15"', "text/html; charset = 'iso-8859-15'"],+ )+ async def test_async_playwright_raw_response_keeps_header_encoding(self, content_type):"""Raw response bytes should still use the charset from Content-Type."""mock_response = make_async_playwright_response()+ mock_response.headers = {"content-type": content_type}+ mock_response.all_headers.return_value = mock_response.headersresponse = await ResponseFactory.from_async_playwright_response(None,@@ -145,30 +158,317 @@ class TestResponseFactory:assert response.encoding == "iso-8859-15"assert "Bürger" in response.html_content- def test_response_history_processing(self):- """Test processing response history"""- # Mock responses with redirects- mock_final = Mock()- mock_final.status = 200- mock_final.status_text = "OK"- mock_final.all_headers = Mock(return_value={})-- mock_redirect = Mock()- mock_redirect.url = "https://example.com/redirect"- mock_redirect.response = Mock(return_value=mock_final)- mock_redirect.all_headers = Mock(return_value={})- mock_redirect.redirected_from = None-- mock_first = Mock()- mock_first.request.redirected_from = mock_redirect-- # Process history- history = ResponseFactory._process_response_history(- mock_first,- {}- )+ @pytest.mark.asyncio+ @pytest.mark.parametrize("asynchronous", [False, True])+ async def test_response_history_processing(self, asynchronous):+ make = make_async_playwright_response if asynchronous else make_sync_playwright_response+ make_method = AsyncMock if asynchronous else Mock+ first, middle, final = make(), make(), make()+ for native, path, status in ((first, "start", 301), (middle, "middle", 302), (final, "final", 200)):+ native.url = native.request.url = f"https://example.com/{path}"+ native.status = status+ native.status_text = ""+ native.request.response = make_method(return_value=native)+ native.request.all_headers.return_value = {"x-redirect": path}+ final.request.redirected_from = middle.request+ middle.request.redirected_from = first.request+ if asynchronous:+ response = await ResponseFactory.from_async_playwright_response(None, final, None, {})+ else:+ response = ResponseFactory.from_playwright_response(None, final, None, {})+ assert response.status == 200 and response.url == "https://example.com/final"+ assert [(item.url, item.status, item.reason) for item in response.history] == [+ ("https://example.com/start", 301, "Moved Permanently"),+ ("https://example.com/middle", 302, "Found"),+ ]+ assert [item.request_headers for item in response.history] == [+ {"x-redirect": "start"},+ {"x-redirect": "middle"},+ ]+ assert all(item.body == b"" and item.encoding == "iso-8859-15" for item in response.history)++[email protected]("asynchronous", [False, True])+class TestRecordedResponseFactory:+ @staticmethod+ async def convert(native, asynchronous, **kwargs):+ if asynchronous:+ return await ResponseFactory.from_async_playwright_response(+ None, native, None, {}, collect_history=False, **kwargs+ )+ return ResponseFactory.from_playwright_response(None, native, None, {}, collect_history=False, **kwargs)++ @staticmethod+ def native(asynchronous, body=b"data", headers=None):+ native = make_async_playwright_response() if asynchronous else make_sync_playwright_response()+ native.headers = {"content-type": "text/plain"} if headers is None else headers+ native.all_headers.return_value = native.headers+ native.body.return_value = body+ return native++ async def test_saved_response_keeps_bytes_and_full_headers(self, asynchronous):+ native = self.native(asynchronous, b'\x00{"data": "saved"}\x00')+ native.all_headers.return_value = {**native.headers, "set-cookie": "session=value"}+ native.request.all_headers.return_value = {"cookie": "session=value"}+ metadata = {"proxy": "http://proxy"}++ response = await self.convert(native, asynchronous, max_body_bytes=100, meta=metadata)- assert len(history) >= 0 # Should process redirects+ assert isinstance(response, Response)+ assert response.body == b'\x00{"data": "saved"}\x00'+ assert response.headers["set-cookie"] == "session=value"+ assert response.request_headers["cookie"] == "session=value"+ assert response.meta == {"proxy": "http://proxy"}+ assert metadata == {"proxy": "http://proxy"}++ @pytest.mark.parametrize("method,status", [("HEAD", 200), ("GET", 204), ("GET", 205), ("GET", 304)])+ @pytest.mark.parametrize("declared_length", [None, "1000000"])+ async def test_absent_bodies_are_not_read(self, asynchronous, method, status, declared_length):+ native = self.native(asynchronous, headers={})+ native.request.method, native.status = method, status+ if declared_length is not None:+ native.headers.update({"content-type": "application/octet-stream", "content-length": declared_length})++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == b""+ assert response.meta == {}+ native.body.assert_not_called()++ async def test_content_length_skips_large_body(self, asynchronous):+ native = self.native(asynchronous, headers={"content-type": "text/plain", "content-length": "101"})++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == b""+ assert response.meta == {"body_note": "too large; not saved."}+ native.body.assert_not_called()++ @pytest.mark.parametrize("content_type", ["application/octet-stream", "image/png", "application/x-custom", None])+ async def test_non_text_bodies_are_not_read_but_metadata_is_saved(self, asynchronous, content_type):+ headers = {} if content_type is None else {"content-type": content_type}+ native = self.native(asynchronous, b"\x00\xffbinary", headers)+ native.status, native.status_text, native.request.method = 206, "Partial Content", "POST"+ native.all_headers.return_value = {**headers, "x-saved": "response"}+ native.request.all_headers.return_value = {"x-saved": "request"}+ native.body.side_effect = AssertionError("Non-text body must not be read")++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == b""+ assert response.meta == {"body_note": "Non-text body; not saved."}+ assert response.url == native.url and response.status == 206 and response.reason == "Partial Content"+ assert response.headers == {**headers, "x-saved": "response"}+ assert response.request_headers == {"x-saved": "request"}+ native.body.assert_not_called()++ @pytest.mark.parametrize(+ "content_type,body",+ [+ (' Text/Plain ; charset="utf-8"', "café".encode()),+ ("application/json", b'{"saved": true}'),+ ("application/xml", b"<saved>true</saved>"),+ ("application/javascript", b"const saved = true;"),+ ("application/x-javascript", b"var saved = true;"),+ ("application/graphql", b"{ saved }"),+ ("application/x-www-form-urlencoded", b"saved=true"),+ ("application/problem+json", b'{"saved": true}'),+ ("image/svg+xml", b"<svg></svg>"),+ ],+ )+ async def test_text_content_types_keep_the_complete_body(self, asynchronous, content_type, body):+ native = self.native(asynchronous, body, {"content-type": content_type})++ response = await self.convert(native, asynchronous, max_body_bytes=len(body))++ assert response.body == body+ assert response.headers == {"content-type": content_type}+ assert response.meta == {}+ native.body.assert_called_once_with()++ @pytest.mark.parametrize("declared_length", [None, "1", "invalid"])+ async def test_actual_size_rejects_whole_body(self, asynchronous, declared_length):+ native = self.native(asynchronous, b"12345")+ if declared_length is not None:+ native.headers["content-length"] = declared_length++ response = await self.convert(native, asynchronous, max_body_bytes=4)++ assert response.body == b""+ assert response.meta == {"body_note": "too large; not saved."}+ native.body.assert_called_once()++ @pytest.mark.parametrize("body,limit", [(b"1234", 4), (b"", 0)])+ @pytest.mark.parametrize("declared", [False, True])+ async def test_available_body_at_limit(self, asynchronous, body, limit, declared):+ native = self.native(asynchronous, body)+ if declared:+ native.headers["content-length"] = str(len(body))++ response = await self.convert(native, asynchronous, max_body_bytes=limit)++ assert response.body == body+ assert "body_note" not in response.meta++ @pytest.mark.parametrize("message", ["Response body is unavailable", ""])+ async def test_read_failure_is_not_an_empty_available_body(self, asynchronous, monkeypatch, message):+ native = self.native(asynchronous)+ native.body.side_effect = RuntimeError(message)+ logger = Mock()+ monkeypatch.setattr("scrapling.engines.toolbelt.convertor.log", logger)++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == b""+ assert response.meta == {"body_note": message or "could not be saved."}+ logger.error.assert_not_called()++ async def test_header_failures_keep_partial_headers(self, asynchronous):+ native = self.native(asynchronous)+ native.all_headers.side_effect = RuntimeError("closed")+ native.request.all_headers.side_effect = RuntimeError("closed")+ native.request.headers = {"accept": "text/plain"}++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == b"data"+ assert response.headers == {"content-type": "text/plain"}+ assert response.request_headers == {"accept": "text/plain"}+ assert response.meta == {"headers_partial": True, "request_headers_partial": True}++ @pytest.mark.parametrize(+ "body,content_type",+ [(b"\xff\x00\xfe", "text/plain"), (b"\xff\x00\xfe", "text/plain; charset=bogus")],+ )+ async def test_parser_errors_do_not_drop_saved_bytes(self, asynchronous, body, content_type):+ native = self.native(asynchronous, body, {"content-type": content_type})++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == body+ assert "body_note" not in response.meta+ assert response.headers["content-type"] == content_type+ assert isinstance(response.html_content, str)+ if content_type.endswith("charset=bogus"):+ assert response.encoding == "utf-8" and response.get_all_text() == ""++ @pytest.mark.parametrize(+ "body,expected,encoding",+ [+ ("café €".encode(), "café €", "utf-8"),+ (b"caf\xe9 \xa4", "café €", "iso-8859-15"),+ (b"\xc3\xa9", "é", "utf-8"),+ ],+ )+ @pytest.mark.parametrize("recorded", [False, True])+ async def test_native_bytes_and_selected_text_encoding(self, asynchronous, body, expected, encoding, recorded):+ content_type = "text/plain; charset=iso-8859-15"+ native = self.native(asynchronous, body, {"content-type": content_type})++ response = await self.convert(native, asynchronous, max_body_bytes=100 if recorded else None)++ assert response.body == body+ assert response.headers["content-type"] == content_type+ assert response.encoding == (encoding if recorded else "iso-8859-15")+ assert response.get_all_text().strip() == (expected if recorded else body.decode("iso-8859-15"))+ assert response.meta == {}+ native.body.assert_called_once_with()++ async def test_recorded_utf8_text_ignores_unknown_declared_charset(self, asynchronous):+ body = '{"saved": "café €"}'.encode()+ content_type = "application/json; charset=bogus"+ native = self.native(asynchronous, body, {"content-type": content_type})++ response = await self.convert(native, asynchronous, max_body_bytes=100)++ assert response.body == body and response.headers["content-type"] == content_type+ assert response.encoding == "utf-8"+ assert response.get_all_text().strip() == '{"saved": "café €"}'+ assert response.json() == {"saved": "café €"}+ assert response.meta == {}++ async def test_unlimited_mode_does_not_add_recording_metadata(self, asynchronous):+ native = self.native(asynchronous, b"data", {"content-length": "10000000"})++ response = await self.convert(native, asynchronous)++ assert response.body == b"data"+ assert response.meta == {}+ native.body.assert_called_once()++ async def test_unlimited_mode_keeps_binary_bodies(self, asynchronous):+ body = b"\x89PNG\r\n\x1a\n\x00binary"+ native = self.native(asynchronous, body, {"content-type": "image/png"})++ response = await self.convert(native, asynchronous)++ assert response.body == body+ assert response.headers == {"content-type": "image/png"}+ assert response.meta == {}+ native.body.assert_called_once_with()++ @pytest.mark.parametrize("source", ["response", "request"])+ async def test_unlimited_mode_keeps_header_errors(self, asynchronous, source):+ native = self.native(asynchronous)+ target = native if source == "response" else native.request+ target.all_headers.side_effect = RuntimeError("closed")++ with pytest.raises(RuntimeError, match="closed"):+ await self.convert(native, asynchronous)++ async def test_unlimited_mode_keeps_body_error_behavior(self, asynchronous, monkeypatch):+ native = self.native(asynchronous)+ native.body.side_effect = RuntimeError("closed")+ logger = Mock()+ monkeypatch.setattr("scrapling.engines.toolbelt.convertor.log", logger)++ response = await self.convert(native, asynchronous)++ assert response.body == b""+ assert response.meta == {}+ logger.error.assert_called_once()++ async def test_unlimited_mode_keeps_encoding_errors(self, asynchronous):+ native = self.native(asynchronous, headers={"content-type": "text/plain; charset=bogus"})++ with pytest.raises(LookupError):+ await self.convert(native, asynchronous)++ @pytest.mark.parametrize("source", ["response", "request", "both"])+ async def test_body_error_stays_unavailable_when_headers_also_fail(self, asynchronous, source):+ native = self.native(asynchronous)+ native.body.side_effect = RuntimeError("Body is gone")+ if source in {"response", "both"}:+ native.all_headers.side_effect = RuntimeError("Response headers are gone")+ if source in {"request", "both"}:+ native.request.all_headers.side_effect = RuntimeError("Request headers are gone")+ response = await self.convert(native, asynchronous, max_body_bytes=100)+ assert response.body == b""+ assert response.meta["body_note"] == "Body is gone"+ assert response.meta.get("headers_partial", False) == (source in {"response", "both"})+ assert response.meta.get("request_headers_partial", False) == (source in {"request", "both"})+ assert response.headers == native.headers and response.request_headers == native.request.headers++ async def test_missing_response_raises_instead_of_returning_empty_data(self, asynchronous):+ with pytest.raises(ValueError, match="Failed to get a response"):+ await self.convert(None, asynchronous, max_body_bytes=100)++ @pytest.mark.parametrize(+ "content_type,body",+ [("application/json", b'{"saved": true}'), ("application/octet-stream", b"\x00\xffbinary")],+ )+ async def test_recording_rules_do_not_change_captured_xhr(self, asynchronous, content_type, body):+ native = self.native(asynchronous)+ xhr = self.native(asynchronous, body, {"content-type": content_type})++ response = await self.convert(native, asynchronous, max_body_bytes=4, xhr_captured=[xhr])++ assert response.body == b"data"+ assert response.captured_xhr[0].body == body+ assert response.captured_xhr[0].headers == {"content-type": content_type}+ assert response.captured_xhr[0].meta == {}+ xhr.body.assert_called_once_with()class TestErrorScenarios:diff --git a/tests/fetchers/test_utils.py b/tests/fetchers/test_utils.pyindex 35eeb85..8f36672 100644--- a/tests/fetchers/test_utils.py+++ b/tests/fetchers/test_utils.py@@ -149,30 +149,46 @@ def test_unknown_status_code():assert StatusText.get(1000) == "Unknown Status Code"-# The private classmethod is name-mangled; resolve it once for the tests below.-_extract_encoding = getattr(ResponseFactory, "_ResponseFactory__extract_browser_encoding")--def test_browser_encoding_unquoted_charset():"""A charset declared without quotes is returned verbatim."""- assert _extract_encoding("text/html; charset=utf-8") == "utf-8"- assert _extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1"- assert _extract_encoding("text/html;charset=windows-1252") == "windows-1252"+ assert ResponseFactory._extract_encoding("text/html; charset=utf-8") == "utf-8"+ assert ResponseFactory._extract_encoding("text/html; charset=ISO-8859-1") == "ISO-8859-1"+ assert ResponseFactory._extract_encoding("text/html;charset=windows-1252") == "windows-1252"def test_browser_encoding_quoted_charset():"""A quoted charset value (RFC 7231 allows quoting) is unwrapped, not dropped."""- assert _extract_encoding('text/html; charset="utf-8"') == "utf-8"- assert _extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1"- assert _extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS"- assert _extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252"+ assert ResponseFactory._extract_encoding('text/html; charset="utf-8"') == "utf-8"+ assert ResponseFactory._extract_encoding('text/html; charset="ISO-8859-1"') == "ISO-8859-1"+ assert ResponseFactory._extract_encoding("text/html; charset='Shift_JIS'") == "Shift_JIS"+ assert ResponseFactory._extract_encoding('text/plain; charset="windows-1252"; boundary=x') == "windows-1252"def test_browser_encoding_defaults_when_missing():"""Fall back to the default when no charset is present or the header is empty."""- assert _extract_encoding("text/html") == "utf-8"- assert _extract_encoding("") == "utf-8"- assert _extract_encoding(None) == "utf-8"+ assert ResponseFactory._extract_encoding("text/html") == "utf-8"+ assert ResponseFactory._extract_encoding("") == "utf-8"+ assert ResponseFactory._extract_encoding(None) == "utf-8"+++ "header, expected",+ [+ ('text/plain; CHARSET = "ISO-8859-1"', "ISO-8859-1"),+ ("text/plain; charset = utf-16", "utf-16"),+ ('text/plain; charset=""', "utf-8"),+ ('text/plain; note="charset=utf-16"', "utf-8"),+ ("text/plain; x-charset=utf-16", "utf-8"),+ ("text/plain; charset*=utf-8''UTF-16", "UTF-16"),+ ],+)+def test_browser_encoding_header_parameters(header, expected):+ assert ResponseFactory._extract_encoding(header) == expected+++def test_browser_encoding_custom_default():+ assert ResponseFactory._extract_encoding(None, "ISO-8859-1") == "ISO-8859-1"+ assert ResponseFactory._extract_encoding('text/plain; charset=""', "ISO-8859-1") == "ISO-8859-1"class TestConstructProxyDict: