🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: adjust the unit tests to the latest changes
Karim shoair49a12d4f8b0f
diff --git a/tests/ai/test_browser_network.py b/tests/ai/test_browser_network.pyindex 712a771..adeb0da 100644--- a/tests/ai/test_browser_network.py+++ b/tests/ai/test_browser_network.py@@ -1,7 +1,6 @@from json import loadsfrom os import getenvfrom asyncio import Event, create_task, sleep, wait_for-from dataclasses import replacefrom types import SimpleNamespacefrom urllib.parse import urlsplitfrom unittest.mock import AsyncMock, Mock@@ -13,13 +12,13 @@ from mcp.types import CallToolResult, TextContentfrom scrapling.core._types import Anyfrom scrapling.core._network_formatting import NetworkRequestModel, _request_detailsfrom scrapling.core.ai import NetworkRequestsModel, ScraplingMCPServer, _SessionEntry-from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest+from scrapling.engines._browsers._network import NetworkRecorderfrom scrapling.engines.toolbelt.custom import Responsedef _server() -> tuple[ScraplingMCPServer, Mock]:server = ScraplingMCPServer()- saved = Response(+ record = Response(url="https://network.test/api/items",content='{"name":"مرحبا"}'.encode(),status=201,@@ -27,17 +26,8 @@ def _server() -> tuple[ScraplingMCPServer, Mock]:cookies={},headers={"content-type": "application/json; charset=utf-8", "set-cookie": "test=2"},request_headers={"content-type": "application/json", "cookie": "test=1"},- )- record = NetworkRequest(- id=4,- url=saved.url,method="POST",- resource_type="fetch",- status=201,- request_headers=saved.request_headers,- request_body=b'{"sent":true}',- response_headers=saved.headers,- response=saved,+ meta={"network_id": 4, "resource_type": "fetch", "request_body": b'{"sent":true}'},)network = Mock(spec=["last_id", "dropped_count", "search", "get"], last_id=4, dropped_count=0)network.search.return_value = [record]@@ -202,12 +192,16 @@ async def test_network_tools_schema_and_content() -> None:async def test_network_list_forwards_filters_and_reports_paging_eviction() -> None:server, network = _server()network.search.return_value.append(- replace(- network.get(4),- id=8,+ Response(url="https://network.test/next",+ content=b"next",+ status=200,+ reason="OK",+ cookies={},+ headers={"content-type": "text/plain"},+ request_headers={},method="GET",- resource_type="document",+ meta={"network_id": 8, "resource_type": "document", "request_body": None},))network.last_id = 8@@ -405,7 +399,7 @@ async def test_network_listing_preserves_url_characters_and_only_returns_summarynetwork._detach()record = network.get(1)assert record is not None and record.url == native.url- assert record.request_body == b'{"secret":"request"}' and record.response.body == b'{"secret":"response"}'+ assert record.meta["request_body"] == b'{"secret":"request"}' and record.body == b'{"secret":"response"}'readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)for reader in readers:reader.assert_awaited_once()@@ -434,12 +428,13 @@ async def test_network_detail_returns_only_selected_part(part: Any) -> None:if part == "summary":assert _detail(result) == _detail_info(4, part, _request_info(4, "/api/items", "POST", status=201))elif part.endswith("headers"):- expected = record.response.request_headers if part == "request_headers" else record.response.headers+ expected = record.request_headers if part == "request_headers" else record.headersassert _detail(result) == _detail_info(4, part, expected)else:expected = '{"sent":true}' if part == "request_body" else '{"name":"مرحبا"}'assert _detail(result) == _detail_info(4, part, expected)- assert not hasattr(record, "request")+ assert isinstance(record, Response) and record.request is None+ assert not hasattr(record, "response")network.search.assert_not_called()@@ -450,7 +445,7 @@ def test_saved_response_details_are_local(part: Any) -> None:result = _request_details(record, part)assert isinstance(result, NetworkRequestModel)if part.endswith("headers"):- expected = record.response.request_headers if part == "request_headers" else record.response.headers+ expected = record.request_headers if part == "request_headers" else record.headersassert result.model_dump() == _detail_info(4, part, expected)else:assert result.model_dump() == _detail_info(4, part, '{"name":"مرحبا"}')@@ -472,7 +467,7 @@ async def test_network_objects_keep_all_summary_fields_and_header_values(part: sif part == "request_headers":record.request_headers = headerselif part == "response_headers":- record.response_headers = headers+ record.headers = headersexpected = _request_info(4, "/api/items", "POST", status=201) if part == "summary" else headersasync with Client(server._build_server("127.0.0.1", 8000)) as client:result = await client.call_tool(@@ -489,9 +484,9 @@ async def test_network_unicode_body_returns_the_full_original_text(part: str) ->record = network.get(4)text = 'Aمرحبا🙂e\u0301\n"z"'if part == "request_body":- record.request_body = text.encode()+ record.meta["request_body"] = text.encode()else:- record.response._raw_body = text.encode()+ record._raw_body = text.encode()async with Client(server._build_server("127.0.0.1", 8000)) as client:result = await client.call_tool("browser_network_request", {"session_id": "browser", "request_id": 4, "part": part}@@ -517,7 +512,7 @@ async def test_network_detail_returns_complete_empty_and_large_saved_bodies(partnetwork._detach()record = network.get(1)assert record is not None- assert (record.request_body if part == "request_body" else record.response.body) == text.encode()+ assert (record.meta["request_body"] if part == "request_body" else record.body) == text.encode()readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)for reader in readers:reader.assert_awaited_once()@@ -537,14 +532,14 @@ async def test_network_detail_body_availability(case: str) -> None:server, network = _server()record = network.get(4)if case == "empty":- record.response.status = 204- record.response._raw_body = b""+ record.status = 204+ record._raw_body = b""elif case == "binary":- record.response.headers["content-type"] = "image/png"+ record.headers["content-type"] = "image/png"elif case == "too_large":- record.response.meta["body_note"] = "too large; not saved."+ record.meta["body_note"] = "too large; not saved."else:- record.response.meta["body_note"] = "Context closed"+ record.meta["body_note"] = "Context closed"result = await server.browser_network_request("browser", 4, "response_body")expected = {"empty": _detail_info(4, "response_body", ""),@@ -561,9 +556,9 @@ async def test_network_detail_body_availability(case: str) -> None:async def test_network_detail_saved_headers_report_completeness(part: Any, partial: bool) -> None:server, network = _server()record = network.get(4)- headers = record.request_headers if part == "request_headers" else record.response_headers+ headers = record.request_headers if part == "request_headers" else record.headerskey = "request_headers_partial" if part == "request_headers" else "headers_partial"- record.response.meta[key] = partial+ record.meta[key] = partialresult = await server.browser_network_request("browser", 4, part)assert result.model_dump() == _detail_info(4, part, headers, note="Headers are partial; full headers unavailable." if partial else None@@ -588,10 +583,10 @@ async def test_network_detail_decodes_text_for_any_resource() -> None:server, network = _server()record = network.get(4)- record.resource_type = "document"- record.response.headers["content-type"] = content_type- record.response._raw_body = body- record.response.encoding = encoding+ record.meta["resource_type"] = "document"+ record.headers["content-type"] = content_type+ record._raw_body = body+ record.encoding = encodingresult = await server.browser_network_request("browser", 4, "response_body")assert result.model_dump() == _detail_info(4, "response_body", expected)@@ -660,9 +655,9 @@ async def test_network_response_uses_saved_encoding_and_request_uses_its_header(network._detach()record = network.get(1)assert record is not None- assert record.response.encoding == response_encoding- assert record.response.body == native_body and record.request_body == request_body- assert record.response_headers == {"content-type": response_type}+ assert record.encoding == response_encoding+ assert record.body == native_body and record.meta["request_body"] == request_body+ assert record.headers == {"content-type": response_type}assert record.request_headers == {"content-type": request_type}readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)for reader in readers:@@ -681,19 +676,18 @@ async def test_network_response_uses_saved_encoding_and_request_uses_its_header(assert _detail(result) == _detail_info(1, part, {"content-type": expected})for reader in readers:reader.assert_awaited_once()- assert record.response.body == native_body and record.request_body == request_body+ assert record.body == native_body and record.meta["request_body"] == request_body@pytest.mark.asyncioasync def test_network_request_body_absent_and_binary() -> None:server, network = _server()record = network.get(4)- record.request_body = None+ record.meta["request_body"] = Noneresult = await server.browser_network_request("browser", 4, "request_body")assert result.model_dump() == _detail_info(4, "request_body", note="Body is absent.")- record.request_body = b"binary"+ record.meta["request_body"] = b"binary"record.request_headers["content-type"] = "application/octet-stream"- record.response.request_headers["content-type"] = "application/octet-stream"result = await server.browser_network_request("browser", 4, "request_body")assert result.model_dump() == _detail_info(4, "request_body", note="Only text bodies can be displayed.")@@ -737,9 +731,14 @@ async def test_network_reads_saved_parts_without_native_access(record = network.get(1)assert record is not Noneskipped = part == "response_body" and note == "Non-text body; not saved."- assert record.response.meta == ({"body_note": note} if skipped else {})+ assert record.meta == {+ "network_id": 1,+ "resource_type": "fetch",+ "request_body": body if part == "request_body" else None,+ **({"body_note": note} if skipped else {}),+ }if part == "response_body":- assert record.response.body == (b"" if skipped else body)+ assert record.body == (b"" if skipped else body)assert native.existing_response.body.await_count == (not skipped and status != 204)readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)counts = [reader.await_count for reader in readers]@@ -759,7 +758,7 @@ async def test_network_reads_saved_parts_without_native_access(if saved_part == "summary"else record.request_headersif saved_part == "request_headers"- else record.response_headers+ else record.headers)assert _detail(result) == _detail_info(1, saved_part, expected_data)assert [reader.await_count for reader in readers] == counts@@ -791,8 +790,13 @@ async def test_network_body_note_does_not_replace_other_saved_parts(error: Excepawait network._contexts[context]["requestfinished"](native)network._detach()record = network.get(1)- assert record is not None and record.response.meta == {"body_note": expected}- assert record.response.body == b"" and record.request_body == b'{"sent":true}'+ assert record is not None and record.meta == {+ "network_id": 1,+ "resource_type": "fetch",+ "request_body": b'{"sent":true}',+ "body_note": expected,+ }+ assert record.body == b"" and record.meta["request_body"] == b'{"sent":true}'assert native.existing_response.body.await_count == (error is not None)readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)counts = [reader.await_count for reader in readers]@@ -890,7 +894,7 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:status, content_type, body = responses.get(path, (200, "text/html", html.encode()))await route.fulfill(status=status, content_type=content_type, body=body, headers={"x-recorded": "yes"})- async def captured(session: Any, pattern: str) -> NetworkRequest:+ async def captured(session: Any, pattern: str) -> Response:while not (records := session.network.search(url_pattern=pattern)):await sleep(0.01)return records[0]@@ -933,8 +937,13 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:}assert session.network.search(url_pattern="/api/failed$") == []for name, record in records.items():- assert record.response.body == (b"" if name == "binary" else responses[f"/api/{name}"][2])- assert record.response.meta == ({"body_note": "Non-text body; not saved."} if name == "binary" else {})+ assert record.body == (b"" if name == "binary" else responses[f"/api/{name}"][2])+ assert record.meta == {+ "network_id": record.meta["network_id"],+ "resource_type": "fetch",+ "request_body": None,+ **({"body_note": "Non-text body; not saved."} if name == "binary" else {}),+ }before = list(seen)page = session.page_pool.get_ready_page()assert page is not None@@ -942,14 +951,14 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:listed = await client.call_tool("browser_network_requests", {"session_id": "browser"})detail = await client.call_tool("browser_network_request",- {"session_id": "browser", "request_id": request.id, "part": "response_body"},+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "response_body"},)assert not listed.is_error and not detail.is_error- assert request.response is not None and request.response.body == b'{"items":[1,2]}'+ assert request is not None and request.body == b'{"items":[1,2]}'expected_requests = sorted(- [_request_info(request.id, "/api/items", "POST")]+ [_request_info(request.meta["network_id"], "/api/items", "POST")]+ [- _request_info(record.id, f"/api/{name}", status=responses[f"/api/{name}"][0])+ _request_info(record.meta["network_id"], f"/api/{name}", status=responses[f"/api/{name}"][0])for name, record in records.items()],key=lambda item: item["id"],@@ -961,12 +970,14 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:"dropped_count": 0,}assert _listing(listed) == expected_listing- assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')+ assert _detail(detail) == _detail_info(request.meta["network_id"], "response_body", '{"items":[1,2]}')request_body = await client.call_tool("browser_network_request",- {"session_id": "browser", "request_id": request.id, "part": "request_body"},+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "request_body"},+ )+ assert _detail(request_body) == _detail_info(+ request.meta["network_id"], "request_body", '{"sent":true}')- assert _detail(request_body) == _detail_info(request.id, "request_body", '{"sent":true}')for name, expected, note in (("empty", "", None),("absent", "", None),@@ -975,15 +986,25 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:):result = await client.call_tool("browser_network_request",- {"session_id": "browser", "request_id": records[name].id, "part": "response_body"},+ {+ "session_id": "browser",+ "request_id": records[name].meta["network_id"],+ "part": "response_body",+ },+ )+ assert _detail(result) == _detail_info(+ records[name].meta["network_id"], "response_body", expected, note=note)- assert _detail(result) == _detail_info(records[name].id, "response_body", expected, note=note)headers = await client.call_tool("browser_network_request",- {"session_id": "browser", "request_id": records[name].id, "part": "response_headers"},+ {+ "session_id": "browser",+ "request_id": records[name].meta["network_id"],+ "part": "response_headers",+ },)assert _detail(headers) == _detail_info(- records[name].id, "response_headers", records[name].response_headers+ records[name].meta["network_id"], "response_headers", records[name].headers)assert _detail(headers)["data"]["x-recorded"] == "yes"assert page.state == "busy" and seen == before@@ -998,15 +1019,16 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:await session.close_pages()before = list(seen)detail = await client.call_tool(- "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}+ "browser_network_request",+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "response_body"},)- assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')+ assert _detail(detail) == _detail_info(request.meta["network_id"], "response_body", '{"items":[1,2]}')for record in records.values():saved = await client.call_tool("browser_network_request",- {"session_id": "browser", "request_id": record.id, "part": "response_headers"},+ {"session_id": "browser", "request_id": record.meta["network_id"], "part": "response_headers"},)- assert _detail(saved) == _detail_info(record.id, "response_headers", record.response_headers)+ assert _detail(saved) == _detail_info(record.meta["network_id"], "response_headers", record.headers)assert _detail(saved)["data"]["x-recorded"] == "yes"assert seen == beforefinally:@@ -1015,10 +1037,11 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:finally:await session.close()assert not session.network._contexts- assert request.response.body == b'{"items":[1,2]}'+ assert request.body == b'{"items":[1,2]}'closed = await client.call_tool("browser_network_requests", {"session_id": "browser"})assert closed.is_errorclosed_detail = await client.call_tool(- "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}+ "browser_network_request",+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "response_body"},)assert closed_detail.is_errordiff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.pyindex db77925..edee3f0 100644--- a/tests/fetchers/test_network_browser.py+++ b/tests/fetchers/test_network_browser.py@@ -21,6 +21,7 @@ from playwright.sync_api import Response as PlaywrightResponsefrom scrapling.core._types import Any, Generatorfrom scrapling.core._network_formatting import _request_detailsfrom scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession+from scrapling.engines.toolbelt.custom import Responsefrom scrapling.engines.toolbelt.proxy_rotation import ProxyRotator@@ -74,6 +75,12 @@ MIME_FETCH = """async paths => {}return bodies;}"""+SEARCH_FETCH = """async () => {+ await Promise.all(Array.from({length: 150}, (_, index) => fetch(`/api/search/${index}`, index % 10 ? {+ method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({index})+ } : {}).then(response => response.json())));+ return true;+}"""CHARSET_BODY = "caf\u00e9 \u20ac".encode("iso-8859-15")CHARSET_PATHS = ("/wire/charset", "/wire/charset-spaced", "/wire/raw-charset")WIRE_PATHS = ["/wire/chunked", "/wire/gzip", "/wire/binary", *CHARSET_PATHS, "/wire/head", "/wire/empty"]@@ -262,7 +269,7 @@ async def test_async_network_disabled_by_default(session_type: Any) -> None:def _wait_saved(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:for _ in range(250):- records = network.search(url_pattern=pattern, after_id=after_id)+ records = network.search(url_pattern=pattern, after_id=after_id, limit=None)if len(records) >= count:return recordspage.wait_for_timeout(20)@@ -271,44 +278,54 @@ def _wait_saved(page: Any, network: Any, pattern: str, count: int = 1, after_id:async def _wait_saved_async(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:for _ in range(250):- records = network.search(url_pattern=pattern, after_id=after_id)+ records = network.search(url_pattern=pattern, after_id=after_id, limit=None)if len(records) >= count:return recordsawait asyncio.sleep(0.02)raise AssertionError(f"Expected {count} saved responses for {pattern!r}, got {len(records)}")-def _check_initial(network: Any, response: Any) -> Any:+def _check_initial(network: Any) -> Any:records = network.search()assert network.enabled- assert {record.resource_type for record in records} >= {"document", "script", "stylesheet", "image", "fetch", "xhr"}- assert len({record.id for record in records}) == len(records)+ assert {record.meta["resource_type"] for record in records} >= {+ "document",+ "script",+ "stylesheet",+ "image",+ "fetch",+ "xhr",+ }+ assert len({record.meta["network_id"] for record in records}) == len(records)initial = network.search(url_pattern=r"/api/initial$", resource_type="fetch")assert len(initial) == 1assert initial[0].status == 200- assert initial[0].request_body is None- assert initial[0].response.json() == {"ok": True}- assert "body_note" not in initial[0].response.meta and not hasattr(initial[0], "request")- assert {urlsplit(item.url).path for item in response.captured_xhr} == {"/api/initial", "/api/xhr"}- assert all(item.json() == {"ok": True} for item in response.captured_xhr)+ assert initial[0].meta["request_body"] is None+ assert initial[0].json() == {"ok": True}+ assert isinstance(initial[0], Response) and initial[0].request is None+ assert "body_note" not in initial[0].meta and not hasattr(initial[0], "response")+ assert network.get(initial[0].meta["network_id"]) is initial[0]+ captured = network.search(url_pattern=r"/api/(initial|xhr)$")+ assert {urlsplit(item.url).path for item in captured} == {"/api/initial", "/api/xhr"}+ assert all(item.json() == {"ok": True} for item in captured)image = network.search(url_pattern=r"/image\.png$")[0]- assert image.response.body == b"" and image.response.meta.get("body_note") == "Non-text body; not saved."+ assert image.body == b"" and image.meta.get("body_note") == "Non-text body; not saved."for path, body in ((r"/script\.js$", SCRIPT), (r"/style\.css$", b"body { color: black; }")):asset = network.search(url_pattern=path)[0]- assert asset.response.body == body and "body_note" not in asset.response.meta+ assert asset.body == body and "body_note" not in asset.metafiltered = network.search(include_static=False)- assert filtered and all(record.resource_type in ("document", "xhr", "fetch") for record in filtered)+ assert filtered and all(record.meta["resource_type"] in ("document", "xhr", "fetch") for record in filtered)return initial[0]def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:records = network.search(after_id=after_id)- assert records and all(record.id > after_id for record in records)+ assert records and all(record.meta["network_id"] > after_id for record in records)post = network.search(method="POST", status=201)assert len(post) == 1- assert loads(post[0].request_body) == {"sent": True}+ assert loads(post[0].meta["request_body"]) == {"sent": True}duplicate = network.search(url_pattern=r"/api/repeated$")- assert len(duplicate) == 2 and duplicate[0].id != duplicate[1].id+ assert len(duplicate) == 2 and duplicate[0].meta["network_id"] != duplicate[1].meta["network_id"]assert len(network.search(url_pattern=r"/api/repeated$", limit=1)) == 1status = network.search(status=503)assert len(status) == 1@@ -316,38 +333,34 @@ def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:redirect = network.search(url_pattern=r"/redirect$")assert len(redirect) == 1 and redirect[0].status == 302redirected = network.search(url_pattern=r"/redirected$", status=200)- assert len(redirected) == 1 and redirected[0].id != redirect[0].id- assert redirect[0].response_headers["location"] == "/redirected"+ assert len(redirected) == 1 and redirected[0].meta["network_id"] != redirect[0].meta["network_id"]+ assert redirect[0].headers["location"] == "/redirected"return post[0], status[0], network.search(url_pattern=r"/api/large$")[0]@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])[email protected]("capture_xhr", [".*", r"/api/"])-def test_sync_network_browser_history(- session_type: Any, network_url: str, release_pending: Event, capture_xhr: str-) -> None:+def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:with session_type(**_options(- capture_xhr=capture_xhr,page_action=lambda page: page.locator("body[data-initial=ready]").wait_for(state="attached"),)) as session:- response = session.fetch(network_url + "/")+ session.fetch(network_url + "/")page = session.page_pool.pages[0].page_wait_saved(page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6)- initial = _check_initial(session.network, response)- assert initial.response.json() == {"ok": True}+ initial = _check_initial(session.network)+ assert initial.json() == {"ok": True}page = session.page_pool.pages[0].pageafter_id = session.network.last_idwith page.expect_request_finished(lambda request: request.url.endswith("/api/large")):page.evaluate(LATER)_wait_saved(page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7)post, status, large = _check_later(session.network, after_id)- assert post.response.request_headers["x-sent"] == "request"- assert post.response.headers["x-recorded"] == "response"- assert post.response.json() == {"sent": True}- assert status.response.body == b"Try later"- assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""+ assert post.request_headers["x-sent"] == "request"+ assert post.headers["x-recorded"] == "response"+ assert post.json() == {"sent": True}+ assert status.body == b"Try later"+ assert large.meta.get("body_note") == "too large; not saved." and large.body == b""release_pending.clear()try:with page.expect_request_finished(lambda request: request.url.endswith("/api/pending")):@@ -356,7 +369,7 @@ def test_sync_network_browser_history(assert not session.network.search(url_pattern=r"/api/pending$")release_pending.set()pending = _wait_saved(page, session.network, r"/api/pending$")[0]- assert pending.status == 200 and pending.response.json() == {"ok": True}+ assert pending.status == 200 and pending.json() == {"ok": True}finally:release_pending.set()other = session.context.new_page()@@ -370,7 +383,7 @@ def test_sync_network_browser_history(session.fetch(network_url + "/second")assert session.page_pool.pages[0].page is pagesession.close_pages()- assert page.is_closed() and session.network.get(post.id).url == post.url+ assert page.is_closed() and session.network.get(post.meta["network_id"]).url == post.urlfresh_id = session.network.last_idsession.fetch(network_url + "/fresh")_wait_saved(session.page_pool.pages[0].page, session.network, r"/(?:fresh|api/initial)$", 2, fresh_id)@@ -383,34 +396,31 @@ def test_sync_network_browser_history(_wait_saved(page, session.network, r"/api/closing$")last_id = session.network.last_idassert session.network.last_id == last_id- assert session.network.get(post.id).status == 201- assert session.network.get(post.id).method == "POST"+ assert session.network.get(post.meta["network_id"]).status == 201+ assert session.network.get(post.meta["network_id"]).method == "POST"closing = session.network.search(url_pattern=r"/api/closing$")[0]- assert closing.response.body == b"closing" and "body_note" not in closing.response.meta- assert unread.response.json() == {"ok": True}- assert initial.response.json() == {"ok": True}- assert post.response.json() == {"sent": True}- assert post.response.request_headers["x-sent"] == "request"- assert post.response.headers["x-recorded"] == "response"+ assert closing.body == b"closing" and "body_note" not in closing.meta+ assert unread.json() == {"ok": True}+ assert initial.json() == {"ok": True}+ assert post.json() == {"sent": True}+ assert post.request_headers["x-sent"] == "request"+ assert post.headers["x-recorded"] == "response"@pytest.mark.asyncio@pytest.mark.parametrize("session_type", [AsyncDynamicSession, AsyncStealthySession])[email protected]("capture_xhr", [".*", r"/api/"])-async def test_async_network_browser_history(- session_type: Any, network_url: str, release_pending: Event, capture_xhr: str-) -> None:+async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:async def initial_page(page: Any) -> None:await page.locator("body[data-initial=ready]").wait_for(state="attached")- async with session_type(**_options(capture_xhr=capture_xhr, page_action=initial_page)) as session:- response = await session.fetch(network_url + "/")+ async with session_type(**_options(page_action=initial_page)) as session:+ await session.fetch(network_url + "/")page = session.page_pool.pages[0].pageawait _wait_saved_async(page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6)- initial = _check_initial(session.network, response)- assert initial.response.json() == {"ok": True}+ initial = _check_initial(session.network)+ assert initial.json() == {"ok": True}page = session.page_pool.pages[0].pageafter_id = session.network.last_idasync with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):@@ -419,11 +429,11 @@ async def test_async_network_browser_history(page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7)post, status, large = _check_later(session.network, after_id)- assert post.response.request_headers["x-sent"] == "request"- assert post.response.headers["x-recorded"] == "response"- assert post.response.json() == {"sent": True}- assert status.response.body == b"Try later"- assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""+ assert post.request_headers["x-sent"] == "request"+ assert post.headers["x-recorded"] == "response"+ assert post.json() == {"sent": True}+ assert status.body == b"Try later"+ assert large.meta.get("body_note") == "too large; not saved." and large.body == b""release_pending.clear()try:async with page.expect_request("**/api/pending"):@@ -431,8 +441,7 @@ async def test_async_network_browser_history(assert not session.network.search(url_pattern=r"/api/pending$")release_pending.set()pending = (await _wait_saved_async(page, session.network, r"/api/pending$"))[0]- assert pending.status == 200 and pending.response.json() == {"ok": True}- assert not any("/api/pending" in record.url for record in response.captured_xhr)+ assert pending.status == 200 and pending.json() == {"ok": True}finally:release_pending.set()other = await session.context.new_page()@@ -446,7 +455,7 @@ async def test_async_network_browser_history(await session.fetch(network_url + "/second")assert session.page_pool.pages[0].page is pageawait session.close_pages()- assert page.is_closed() and session.network.get(post.id).url == post.url+ assert page.is_closed() and session.network.get(post.meta["network_id"]).url == post.urlfresh_id = session.network.last_idawait session.fetch(network_url + "/fresh")await _wait_saved_async(@@ -463,15 +472,15 @@ async def test_async_network_browser_history(await _wait_saved_async(page, session.network, r"/api/closing$")last_id = session.network.last_idassert session.network.last_id == last_id- assert session.network.get(post.id).status == 201- assert session.network.get(post.id).method == "POST"+ assert session.network.get(post.meta["network_id"]).status == 201+ assert session.network.get(post.meta["network_id"]).method == "POST"closing = session.network.search(url_pattern=r"/api/closing$")[0]- assert closing.response.body == b"closing" and "body_note" not in closing.response.meta- assert unread.response.json() == {"ok": True}- assert initial.response.json() == {"ok": True}- assert post.response.json() == {"sent": True}- assert post.response.request_headers["x-sent"] == "request"- assert post.response.headers["x-recorded"] == "response"+ assert closing.body == b"closing" and "body_note" not in closing.meta+ assert unread.json() == {"ok": True}+ assert initial.json() == {"ok": True}+ assert post.json() == {"sent": True}+ assert post.request_headers["x-sent"] == "request"+ assert post.headers["x-recorded"] == "response"def _check_eviction(network: Any) -> int:@@ -500,8 +509,8 @@ def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url:page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")_wait_saved(page, session.network, r"/api/post$")records = session.network.search()- assert len(records) == 1 and records[0].id > last_id- assert records[0].request_body == records[0].response.body == b"fresh"+ assert len(records) == 1 and records[0].meta["network_id"] > last_id+ assert records[0].meta["request_body"] == records[0].body == b"fresh"@pytest.mark.asyncio@@ -521,8 +530,8 @@ async def test_async_network_browser_eviction_and_clear(session_type: Any, netwoawait page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")await _wait_saved_async(page, session.network, r"/api/post$")records = session.network.search()- assert len(records) == 1 and records[0].id > last_id- assert records[0].request_body == records[0].response.body == b"fresh"+ assert len(records) == 1 and records[0].meta["network_id"] > last_id+ assert records[0].meta["request_body"] == records[0].body == b"fresh"PROXY_HTML = '<html><body><script>fetch("/api").then(response => response.json()).then(value => document.body.dataset.api = JSON.stringify(value));</script></body></html>'@@ -549,10 +558,10 @@ def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> Nonesession.fetch("https://network.test/")assert not session.page_pool.pagesrecord = session.network.search(url_pattern=r"/api$")[0]- assert record.status == 200 and record.resource_type == "fetch"- assert record.response.json() == {"temporary": True}- assert session.network.get(record.id).url == "https://network.test/api"- assert record.response.json() == {"temporary": True}+ assert record.status == 200 and record.meta["resource_type"] == "fetch"+ assert record.json() == {"temporary": True}+ assert session.network.get(record.meta["network_id"]).url == "https://network.test/api"+ assert record.json() == {"temporary": True}@pytest.mark.asyncio@@ -577,10 +586,10 @@ async def test_async_network_browser_temporary_proxy_context(session_type: Any)await session.fetch("https://network.test/")assert not session.page_pool.pagesrecord = session.network.search(url_pattern=r"/api$")[0]- assert record.status == 200 and record.resource_type == "fetch"- assert record.response.json() == {"temporary": True}- assert session.network.get(record.id).url == "https://network.test/api"- assert record.response.json() == {"temporary": True}+ assert record.status == 200 and record.meta["resource_type"] == "fetch"+ assert record.json() == {"temporary": True}+ assert session.network.get(record.meta["network_id"]).url == "https://network.test/api"+ assert record.json() == {"temporary": True}def _check_saved_override(network: Any, network_url: str) -> None:@@ -588,9 +597,8 @@ def _check_saved_override(network: Any, network_url: str) -> None:assert len(records) == 1record = records[0]assert record.url == network_url + "/api/overridden"- assert record.request_body == record.response.body == b"overridden"+ assert record.meta["request_body"] == record.body == b"overridden"assert record.request_headers["x-overridden"] == "yes"- assert record.response.request_headers["x-overridden"] == "yes"assert not network.search(url_pattern=r"/api/original$")@@ -670,7 +678,7 @@ def test_sync_network_close_during_unfinished_response(assert monotonic() - started < 5assert not gate.release.is_set() and not gate.finished.is_set()assert page.is_closed()- assert session.network.get(saved.id).response.body == BLANK+ assert session.network.get(saved.meta["network_id"]).body == BLANKassert not session.network.search(url_pattern=r"/api/(?:held|fail)")assert bool(session.network._contexts) == (target == "page")finally:@@ -678,7 +686,7 @@ def test_sync_network_close_during_unfinished_response(assert gate.finished.wait(5), "The server did not finish the released response"assert not session.network._contextsassert session.network.search() == [saved]- assert saved.response.body == BLANK+ assert saved.body == BLANK@pytest.mark.asyncio@@ -706,7 +714,7 @@ async def test_async_network_close_during_unfinished_response(await asyncio.wait_for(close(), timeout=5)assert not gate.release.is_set() and not gate.finished.is_set()assert page.is_closed()- assert session.network.get(saved.id).response.body == BLANK+ assert session.network.get(saved.meta["network_id"]).body == BLANKassert not session.network.search(url_pattern=r"/api/(?:held|fail)")assert bool(session.network._contexts) == (target == "page")finally:@@ -714,19 +722,19 @@ async def test_async_network_close_during_unfinished_response(assert await asyncio.to_thread(gate.finished.wait, 5), "The server did not finish the released response"assert not session.network._contextsassert session.network.search() == [saved]- assert saved.response.body == BLANK+ assert saved.body == BLANKdef _check_burst(network: Any, cursor: int, path: str, expected_paths: list[str]) -> int:newer = network.search(after_id=cursor, limit=1)assert len(newer) == 1 and urlsplit(newer[0].url).path == path- assert newer[0].id == cursor + 1+ assert newer[0].meta["network_id"] == cursor + 1records = network.search(limit=2)- records += network.search(after_id=records[-1].id, limit=2)+ records += network.search(after_id=records[-1].meta["network_id"], limit=2)assert [urlsplit(record.url).path for record in records] == expected_paths[-3:]- assert all(record.response.json() == {"ok": True} for record in records)+ assert all(record.json() == {"ok": True} for record in records)assert network.dropped_count == max(0, len(expected_paths) - 3)- return newer[0].id+ return newer[0].meta["network_id"]@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])@@ -764,7 +772,7 @@ def test_sync_network_burst_completion_order_and_pagination(assert not session.network._contextsrecords = session.network.search()assert [urlsplit(record.url).path for record in records] == completed[-3:]- assert all(record.response.json() == {"ok": True} for record in records)+ assert all(record.json() == {"ok": True} for record in records)@pytest.mark.asyncio@@ -805,7 +813,7 @@ async def test_async_network_burst_completion_order_and_pagination(assert not session.network._contextsrecords = session.network.search()assert [urlsplit(record.url).path for record in records] == completed[-3:]- assert all(record.response.json() == {"ok": True} for record in records)+ assert all(record.json() == {"ok": True} for record in records)def _check_wire_bodies(@@ -814,29 +822,29 @@ def _check_wire_bodies(records = {urlsplit(record.url).path: record for record in network.search(url_pattern=r"/wire/")}assert set(records) == set(WIRE_PATHS)for path in ("/wire/chunked", "/wire/gzip"):- assert records[path].response.meta.get("body_note") == "too large; not saved."- assert records[path].response.body == b""- assert "content-length" not in records["/wire/chunked"].response_headers- assert records["/wire/chunked"].response_headers["transfer-encoding"] == "chunked"- assert int(records["/wire/gzip"].response_headers["content-length"]) < 1024 * 1024- assert records["/wire/gzip"].response_headers["content-encoding"] == "gzip"+ assert records[path].meta.get("body_note") == "too large; not saved."+ assert records[path].body == b""+ assert "content-length" not in records["/wire/chunked"].headers+ assert records["/wire/chunked"].headers["transfer-encoding"] == "chunked"+ assert int(records["/wire/gzip"].headers["content-length"]) < 1024 * 1024+ assert records["/wire/gzip"].headers["content-encoding"] == "gzip"assert bytes(wire["/wire/binary"]) == native_bodies["/wire/binary"] == BINARYfor path in ("/wire/binary", "/wire/raw-charset"):- assert records[path].response.body == b""- assert records[path].response.meta.get("body_note") == "Non-text body; not saved."+ assert records[path].body == b""+ assert records[path].meta.get("body_note") == "Non-text body; not saved."details = _request_details(records[path], "response_body")assert details.data is None and details.note == "Non-text body; not saved."for path in CHARSET_PATHS:assert bytes(wire[path]) == CHARSET_BODY- assert records[path].response_headers["content-type"].lower().endswith(("iso-8859-15", '"iso-8859-15"'))+ assert records[path].headers["content-type"].lower().endswith(("iso-8859-15", '"iso-8859-15"'))for path, text in native_texts.items():- assert records[path].response.body == native_bodies[path]- assert "body_note" not in records[path].response.meta- assert records[path].response.encoding == "utf-8"+ assert records[path].body == native_bodies[path]+ assert "body_note" not in records[path].meta+ assert records[path].encoding == "utf-8"assert native_bodies[path].decode("utf-8") == text- assert records[path].response.get_all_text().strip() == text+ assert records[path].get_all_text().strip() == textdetails = _request_details(records[path], "response_body")- assert details.data == text and details.request_id == records[path].id+ assert details.data == text and details.request_id == records[path].meta["network_id"]assert details.part == "response_body" and details.note is Noneassert native_texts["/wire/charset"] == "caf\u00e9 \u20ac"assert native_bodies["/wire/charset"] == "caf\u00e9 \u20ac".encode("utf-8")@@ -844,7 +852,7 @@ def _check_wire_bodies(assert records["/wire/head"].method == "HEAD" and records["/wire/head"].status == 200assert records["/wire/empty"].status == 204for path in ("/wire/head", "/wire/empty"):- assert records[path].response.body == b"" and "body_note" not in records[path].response.meta+ assert records[path].body == b"" and "body_note" not in records[path].meta@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])@@ -891,17 +899,17 @@ def _check_mime_bodies(network: Any, wire: dict[str, list[int]], reads: list[strfor path, (content_type, body) in MIME_RESPONSES.items():record = records[path]assert bytes(wire[path]) == body- assert record.status == 200 and record.method == "GET" and record.resource_type == "fetch"- assert record.response_headers.get("content-type") == content_type- assert record.response_headers["x-recorded"] == "response"- assert int(record.response_headers["content-length"]) == len(body)+ assert record.status == 200 and record.method == "GET" and record.meta["resource_type"] == "fetch"+ assert record.headers.get("content-type") == content_type+ assert record.headers["x-recorded"] == "response"+ assert int(record.headers["content-length"]) == len(body)if path in TEXT_RESPONSES:- assert record.response.body == body and "body_note" not in record.response.meta+ assert record.body == body and "body_note" not in record.metaelse:- assert record.response.body == b""- assert record.response.meta.get("body_note") == "Non-text body; not saved."+ assert record.body == b""+ assert record.meta.get("body_note") == "Non-text body; not saved."for path in ("/mime/json", "/mime/vendor-json"):- assert records[path].response.json() == {"saved": True}+ assert records[path].json() == {"saved": True}@pytest.mark.parametrize(@@ -956,3 +964,61 @@ async def test_async_network_reads_only_text_bodies(_check_mime_bodies(session.network, wire, reads)assert not session.network._contexts_check_mime_bodies(session.network, wire, reads)+++def _check_unlimited_search(network: Any, after_id: int) -> None:+ records = network.search(limit=None)+ assert len(records) == 125 and network.dropped_count == 25 and network.last_id == after_id + 150+ assert [record.meta["network_id"] for record in records] == list(range(after_id + 26, after_id + 151))+ assert network.get(after_id + 25) is None+ assert network.search() == records[:100]+ for record in records:+ index = int(urlsplit(record.url).path.rsplit("/", 1)[-1])+ assert isinstance(record, Response) and network.get(record.meta["network_id"]) is record+ assert record.request is None and record.meta["resource_type"] == "fetch"+ assert record.method == ("POST" if index % 10 else "GET")+ assert record.status == (201 if index % 10 else 200)+ assert record.json() == ({"index": index} if index % 10 else {"ok": True})+ filtered = network.search(+ url_pattern=r"/api/search/\d+$",+ method="post",+ status=201,+ resource_type="fetch",+ include_static=False,+ limit=None,+ )+ assert len(filtered) > 100 and filtered == [record for record in records if record.method == "POST"]+ records.clear()+ filtered.clear()+ assert len(network.search(limit=None)) == 125 and len(network.search()) == 100++[email protected]("session_type", [DynamicSession, StealthySession])+def test_sync_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:+ with session_type(**_options(max_recorded_requests=125)) as session:+ session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ _wait_saved(page, session.network, r"/blank$")+ after_id = session.network.last_id+ session.network.clear()+ assert page.evaluate(SEARCH_FETCH) is True+ _wait_saved(page, session.network, r"/api/search/", after_id=after_id + 149)+ _check_unlimited_search(session.network, after_id)+ assert not session.network._contexts+ _check_unlimited_search(session.network, after_id)++[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])+async def test_async_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:+ async with session_type(**_options(max_recorded_requests=125)) as session:+ await session.fetch(network_url + "/blank")+ page = session.page_pool.pages[0].page+ await _wait_saved_async(page, session.network, r"/blank$")+ after_id = session.network.last_id+ session.network.clear()+ assert await page.evaluate(SEARCH_FETCH) is True+ await _wait_saved_async(page, session.network, r"/api/search/", after_id=after_id + 149)+ _check_unlimited_search(session.network, after_id)+ assert not session.network._contexts+ _check_unlimited_search(session.network, after_id)diff --git a/tests/fetchers/test_network_cleanup.py b/tests/fetchers/test_network_cleanup.pyindex dc67a75..f7cfc7a 100644--- a/tests/fetchers/test_network_cleanup.py+++ b/tests/fetchers/test_network_cleanup.py@@ -30,8 +30,10 @@ def _closed(session, context, browser, playwright):def test_sync_session_close_during_capture_does_not_restore_late_response():session, context, browser, playwright, capture = _session()+ assert isinstance(session, SyncSession)capture(request(body=b"saved"))saved = session.network.get(1)+ assert saved is not Nonedef close_before_body_returns():session.close()@@ -42,7 +44,7 @@ def test_sync_session_close_during_capture_does_not_restore_late_response():capture(native)session.close()_closed(session, context, browser, playwright)- assert session.network.search() == [saved] and saved.response.body == b"saved"+ assert session.network.search() == [saved] and saved.body == b"saved"assert session.network.last_id == 1@@ -50,8 +52,10 @@ def test_sync_session_close_during_capture_does_not_restore_late_response():@pytest.mark.parametrize("read_error", [False, True])async def test_async_session_close_does_not_wait_for_running_capture(read_error):session, context, browser, playwright, capture = _session(asynchronous=True)+ assert isinstance(session, AsyncSession)await capture(request(body=b"saved", asynchronous=True))saved = session.network.get(1)+ assert saved is not Nonereading, release = asyncio.Event(), asyncio.Event()async def held_body():@@ -75,7 +79,7 @@ async def test_async_session_close_does_not_wait_for_running_capture(read_error)release.set()await asyncio.wait_for(task, timeout=2)await session.close()- assert session.network.search() == [saved] and saved.response.body == b"saved"+ assert session.network.search() == [saved] and saved.body == b"saved"assert session.network.last_id == 1finally:release.set()diff --git a/tests/fetchers/test_network_history.py b/tests/fetchers/test_network_history.pyindex 1743b0b..568f4ac 100644--- a/tests/fetchers/test_network_history.py+++ b/tests/fetchers/test_network_history.py@@ -8,7 +8,7 @@ import pytestfrom scrapling.core._types import Any, Optionalfrom scrapling.engines._browsers import _network-from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest+from scrapling.engines._browsers._network import NetworkRecorderfrom scrapling.engines.toolbelt.custom import Response@@ -81,12 +81,14 @@ async def test_only_completed_responses_are_saved(asynchronous):assert network.search() == []await finish(context, native)record = network.get(1)- assert isinstance(record, NetworkRequest) and not hasattr(record, "request")+ assert isinstance(record, Response) and record.request is Noneassert not hasattr(record, "state") and not hasattr(record, "failure")- assert record.id == 1 and record.url == native.url and record.method == "GET"- assert record.resource_type == "fetch" and record.status == 200- assert record.response.json() == {"ok": True}- assert "body_note" not in record.response.meta+ assert not hasattr(record, "response")+ assert record.meta["network_id"] == 1 and record.url == native.url and record.method == "GET"+ assert record.meta["resource_type"] == "fetch" and record.status == 200+ assert record.json() == {"ok": True}+ assert record.meta == {"network_id": 1, "resource_type": "fetch", "request_body": None}+ assert network.search()[0] is record and network.get(record.meta["network_id"]) is record@pytest.mark.asyncio@@ -104,15 +106,15 @@ async def test_saved_response_has_final_request_data_and_survives_close(asynchroawait finish(context, native)record = network.get(1)assert record is not None- assert record.url == native.url and record.response.url == native.url and record.method == "POST"- assert record.request_body == b"overridden"- assert record.request_headers == record.response.request_headers == native.headers- assert record.response_headers == record.response.headers == native.existing_response.headers+ assert record.url == native.url and record.method == "POST"+ assert record.meta["request_body"] == b"overridden"+ assert record.request_headers == native.headers+ assert record.headers == native.existing_response.headersassert network.search(url_pattern=original_url) == [] and network.search(method="POST") == [record]native.headers["x-overridden"] = "changed"native.existing_response.headers["content-type"] = "changed"assert record.request_headers["x-overridden"] == "yes"- assert record.response_headers["content-type"] == "application/json"+ assert record.headers["content-type"] == "application/json"native.all_headers.assert_called_once_with()native.existing_response.all_headers.assert_called_once_with()native.existing_response.body.assert_called_once_with()@@ -121,7 +123,7 @@ async def test_saved_response_has_final_request_data_and_survives_close(asynchronative.all_headers.side_effect = RuntimeError("Browser closed")native.existing_response.all_headers.side_effect = RuntimeError("Browser closed")native.existing_response.body.side_effect = RuntimeError("Browser closed")- assert record.response.json() == {"ok": True}+ assert record.json() == {"ok": True}assert network.get(1) is record and network.search() == [record]native.existing_response.body.assert_called_once_with()@@ -145,16 +147,18 @@ async def test_recording_skips_fetch_log_without_muting_normal_responses(monkeypawait finish(context, native)record = network.get(1)assert record is not None- assert (- record.response.meta- == {+ assert record.meta == {+ "network_id": 1,+ "resource_type": "fetch",+ "request_body": None,+ **{"saved": {},"oversized": {"body_note": "too large; not saved."},"unreadable": {"body_note": "Body missing"},"invalid_charset": {},"non_text": {"body_note": "Non-text body; not saved."},- }[outcome]- )+ }[outcome],+ }logger.info.assert_not_called()response = Response(url="https://example.test/page",@@ -178,8 +182,13 @@ async def test_oversized_response_keeps_headers_without_reading_body(monkeypatchawait finish(context, native)record = network.get(1)assert record is not None- assert record.status == 200 and record.response.meta == {"body_note": "too large; not saved."}- assert record.response.body == b"" and record.response.headers["content-length"] == "5"+ assert record.status == 200 and record.meta == {+ "network_id": 1,+ "resource_type": "fetch",+ "request_body": None,+ "body_note": "too large; not saved.",+ }+ assert record.body == b"" and record.headers["content-length"] == "5"native.existing_response.body.assert_not_called()@@ -194,8 +203,13 @@ async def test_unreadable_response_keeps_headers_and_reports_unavailable(asynchrrecord = network.get(1)assert record is not Noneassert record.status == 200- assert record.response.meta == {"body_note": message or "could not be saved."}- assert record.response.body == b"" and record.response.headers == native.existing_response.headers+ assert record.meta == {+ "network_id": 1,+ "resource_type": "fetch",+ "request_body": None,+ "body_note": message or "could not be saved.",+ }+ assert record.body == b"" and record.headers == native.existing_response.headers@pytest.mark.asyncio@@ -224,16 +238,21 @@ async def test_non_text_records_keep_metadata_without_using_body_budget(monkeyparecord = network.get(request_id)assert record is not Noneassert record.url == native.url and record.method == "POST" and record.status == 206- assert record.resource_type == "image" and record.request_body == b"submitted"- assert record.request_headers == native.headers and record.response_headers == response.headers- assert record.response.body == b"" and record.response.meta == {"body_note": "Non-text body; not saved."}+ assert record.meta["resource_type"] == "image" and record.meta["request_body"] == b"submitted"+ assert record.request_headers == native.headers and record.headers == response.headers+ assert record.body == b"" and record.meta == {+ "network_id": request_id,+ "resource_type": "image",+ "request_body": b"submitted",+ "body_note": "Non-text body; not saved.",+ }response.body.assert_not_called()await finish(context, request(body=b"5678", asynchronous=asynchronous))- assert [record.id for record in network.search()] == [1, 2, 3, 4, 5]- assert [record.response.body for record in network.search()] == [b"1234", b"", b"", b"", b"5678"]+ assert [record.meta["network_id"] for record in network.search()] == [1, 2, 3, 4, 5]+ assert [record.body for record in network.search()] == [b"1234", b"", b"", b"", b"5678"]assert network.dropped_count == 0await finish(context, request(body=b"9", asynchronous=asynchronous))- assert [record.id for record in network.search()] == [2, 3, 4, 5, 6]+ assert [record.meta["network_id"] for record in network.search()] == [2, 3, 4, 5, 6]assert network.get(1) is None and network.dropped_count == 1@@ -244,15 +263,15 @@ async def test_total_body_budget_evicts_oldest_records_and_clear_releases_budgetnetwork, context = recorder(asynchronous=asynchronous)for body in (b"one", b"two", b"three"):await finish(context, request(body=body, asynchronous=asynchronous))- assert [record.id for record in network.search()] == [3]+ assert [record.meta["network_id"] for record in network.search()] == [3]record = network.get(3)assert record is not None- assert network.dropped_count == 2 and record.response.body == b"three"+ assert network.dropped_count == 2 and record.body == b"three"network.clear()assert network.dropped_count == 0 and network.last_id == 3 and not network.search()for _ in range(2):await finish(context, request(body=b"new", asynchronous=asynchronous))- assert [record.id for record in network.search()] == [4, 5] and network.dropped_count == 0+ assert [record.meta["network_id"] for record in network.search()] == [4, 5] and network.dropped_count == 0@pytest.mark.asyncio@@ -262,13 +281,13 @@ async def test_count_eviction_releases_saved_body_budget(monkeypatch, asynchrononetwork, context = recorder(max_requests=1, asynchronous=asynchronous)for _ in range(3):await finish(context, request(body=b"new", asynchronous=asynchronous))- assert [record.id for record in network.search()] == [3]+ assert [record.meta["network_id"] for record in network.search()] == [3]record = network.get(3)assert record is not None- assert record.response.body == b"new" and network.dropped_count == 2+ assert record.body == b"new" and network.dropped_count == 2network.clear()await finish(context, request(body=b"new", asynchronous=asynchronous))- assert [record.id for record in network.search()] == [4] and network.dropped_count == 0+ assert [record.meta["network_id"] for record in network.search()] == [4] and network.dropped_count == 0@pytest.mark.asyncio@@ -302,7 +321,7 @@ async def test_late_callback_cannot_publish_after_clear_or_detach(change, starteawait finish(context, request(body=b"new", asynchronous=True))record = network.get(1)assert record is not None- assert record.response.body == b"new"+ assert record.body == b"new"finally:release.set()await asyncio.gather(*tasks)@@ -328,10 +347,10 @@ async def test_records_are_published_in_save_order_without_waiting_for_other_bodawait finish(context, request(body=b"second", asynchronous=True))record = network.get(1)assert record is not None- assert record.response.body == b"second"+ assert record.body == b"second"release.set()await asyncio.wait_for(asyncio.gather(*tasks), timeout=2)- assert [record.response.body for record in network.search()] == [b"second", b"first"]+ assert [record.body for record in network.search()] == [b"second", b"first"]finally:release.set()await asyncio.gather(*tasks)@@ -374,13 +393,61 @@ def test_search_filters_include_completed_http_errors():request(url="https://example.test/api", resource_type="xhr"),):context.emit("requestfinished", native)- assert [record.id for record in network.search(include_static=False)] == [2, 3, 4]- assert [record.id for record in network.search(after_id=2, limit=1)] == [3]- assert [record.id for record in network.search(url_pattern="/api$", method="post", status=503)] == [2]- assert [record.id for record in network.search(resource_type="xhr")] == [4]+ assert [record.meta["network_id"] for record in network.search(include_static=False)] == [2, 3, 4]+ assert [record.meta["network_id"] for record in network.search(after_id=2, limit=1)] == [3]+ assert [record.meta["network_id"] for record in network.search(url_pattern="/api$", method="post", status=503)] == [+ 2+ ]+ assert [record.meta["network_id"] for record in network.search(resource_type="xhr")] == [4]assert network.get(100) is None and network.search(url_pattern="absent") == [][email protected]("asynchronous", [False, True])+async def test_unlimited_search_returns_all_retained_matches_and_an_independent_list(asynchronous):+ network, context = recorder(max_requests=250, asynchronous=asynchronous)+ for request_id in range(1, 331):+ native = request(+ url=f"https://example.test/{'api' if request_id % 2 == 0 else 'page'}/{request_id}",+ method="POST" if request_id % 2 == 0 else "GET",+ resource_type="xhr" if request_id % 2 == 0 else "document",+ status=503 if request_id % 2 == 0 else 200,+ body=f'{{"index":{request_id}}}'.encode(),+ asynchronous=asynchronous,+ )+ await finish(context, native)+ records = network.search(limit=None)+ assert len(records) == 250 and network.dropped_count == 80 and network.last_id == 330+ assert [record.meta["network_id"] for record in records] == list(range(81, 331))+ assert network.get(80) is None+ assert network.search() == records[:100]+ assert network.search(limit=7) == records[:7]+ assert all(isinstance(record, Response) and network.get(record.meta["network_id"]) is record for record in records)+ assert all(record.request is None and record.json() == {"index": index} for index, record in enumerate(records, 81))+ filtered = network.search(+ url_pattern=r"/api/\d+$",+ method="post",+ resource_type="xhr",+ status=503,+ after_id=100,+ include_static=False,+ limit=None,+ )+ assert len(filtered) == 115+ assert [record.meta["network_id"] for record in filtered] == list(range(102, 331, 2))+ assert network.search(url_pattern="absent", limit=None) == []+ assert network.search(after_id=330, limit=None) == []+ records.clear()+ filtered.reverse()+ filtered.pop()+ assert len(network.search(limit=None)) == 250 and network.get(330) is not None+ assert [record.meta["network_id"] for record in network.search(resource_type="xhr", limit=None)] == list(+ range(82, 331, 2)+ )+ network.clear()+ assert network.search(limit=None) == [] and network.get(330) is None++@pytest.mark.parametrize("options", [{"limit": 0}, {"after_id": -1}, {"url_pattern": "["}])def test_invalid_filters_raise_value_error(options):with pytest.raises(ValueError):@@ -428,7 +495,7 @@ async def test_native_handles_are_not_kept_in_history(asynchronous, outcome):record = network.get(1)assert record is not Noneassert record.status == 200- assert record.response.meta.get("body_note") == ("Body missing" if outcome == "body_error" else None)+ assert record.meta.get("body_note") == ("Body missing" if outcome == "body_error" else None)del nativegc.collect()assert native_ref() is None and response_ref() is None@@ -454,7 +521,7 @@ def test_sync_conversion_cannot_restore_history_after_reentrant_reset(change):native.existing_response.body.side_effect = bodycontext.emit("requestfinished", native)expected = [] if change == "detach" else [b"new"]- assert [record.response.body for record in network.search()] == expected+ assert [record.body for record in network.search()] == expectedassert network.last_id == len(expected)@@ -480,7 +547,7 @@ async def test_cancelled_conversion_publishes_nothing_and_later_requests_still_sawait finish(context, request(body=b"saved", asynchronous=True))record = network.get(1)assert record is not None- assert record.response.body == b"saved"+ assert record.body == b"saved"finally:release.set()await asyncio.gather(*tasks, return_exceptions=True)@@ -496,10 +563,10 @@ async def test_exact_storage_boundaries_keep_records_until_exceeded(monkeypatch,network, context = recorder(max_requests=3 if budget == "body" else 2, asynchronous=asynchronous)for body in (b"1111", b"2222"):await finish(context, request(body=body, asynchronous=asynchronous))- assert [record.response.body for record in network.search()] == [b"1111", b"2222"]+ assert [record.body for record in network.search()] == [b"1111", b"2222"]assert network.dropped_count == 0await finish(context, request(body=b"3", asynchronous=asynchronous))- assert [record.response.body for record in network.search()] == [b"2222", b"3"]+ assert [record.body for record in network.search()] == [b"2222", b"3"]assert network.get(1) is None and network.dropped_count == 1@@ -517,7 +584,7 @@ async def test_unreadable_request_metadata_is_omitted_and_native_handles_are_relgc.collect()assert native_ref() is None and response_ref() is Noneawait finish(context, request(body=b"next", asynchronous=asynchronous))- assert [record.response.body for record in network.search()] == [b"next"]+ assert [record.body for record in network.search()] == [b"next"]@pytest.mark.asyncio@@ -552,8 +619,8 @@ async def test_saved_native_bytes_and_text_encoding_remain_valid_after_detach(asnetwork._detach()record = network.get(1)assert record is not None- assert record.response.body == body- assert record.response_headers["content-type"] == content_type- assert record.response.encoding == encoding- assert record.response.get_all_text().strip() == expected- assert "body_note" not in record.response.meta+ assert record.body == body+ assert record.headers["content-type"] == content_type+ assert record.encoding == encoding+ assert record.get_all_text().strip() == expected+ assert "body_note" not in record.metadiff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.pyindex 152d525..aedc2b6 100644--- a/tests/fetchers/test_response_handling.py+++ b/tests/fetchers/test_response_handling.py@@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, Mockimport pytestfrom scrapling.parser import Selector+from scrapling.engines._browsers._base import AsyncSession, SyncSessionfrom scrapling.engines.toolbelt.convertor import ResponseFactory, Response@@ -59,6 +60,54 @@ def make_async_page():class TestResponseFactory:"""Test ResponseFactory functionality"""+ @pytest.mark.parametrize("arguments,expected", [({}, "GET"), ({"method": "POST"}, "POST")])+ def test_response_keeps_request_method(self, arguments, expected):+ response = Response(+ url="https://example.com",+ content=b"saved",+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={},+ **arguments,+ )++ assert response.method == expected+ assert not hasattr(response, "captured_xhr")++ @pytest.mark.asyncio+ @pytest.mark.parametrize("asynchronous", [False, True])+ async def test_response_handler_keeps_only_main_frame_navigation(self, asynchronous):+ main_frame = Mock()+ page_info = Mock(page=Mock(main_frame=main_frame))+ responses = [None]+ latest = None+ handler = (+ AsyncSession._create_response_handler(page_info, responses)+ if asynchronous+ else SyncSession._create_response_handler(page_info, responses)+ )+ for resource_type, navigation, main, captured in (+ ("document", True, True, True),+ ("xhr", False, True, False),+ ("fetch", False, True, False),+ ("image", False, True, False),+ ("document", False, True, False),+ ("document", True, False, False),+ ("document", True, True, True),+ ):+ response = Mock()+ response.request.resource_type = resource_type+ response.request.is_navigation_request.return_value = navigation+ response.request.frame = main_frame if main else Mock()+ result = handler(response)+ if result is not None:+ await result+ if captured:+ latest = response+ assert responses[0] is latest+def test_response_from_curl(self):"""Test creating response from curl_cffi response"""# Mock curl response@@ -454,21 +503,11 @@ class TestRecordedResponseFactory:with pytest.raises(ValueError, match="Failed to get a response"):await self.convert(None, asynchronous, max_body_bytes=100)- @pytest.mark.parametrize(- "content_type,body",- [("application/json", b'{"saved": true}'), ("application/octet-stream", b"\x00\xffbinary")],- )- async def test_recording_rules_do_not_change_captured_xhr(self, asynchronous, content_type, body):+ async def test_factory_rejects_removed_xhr_capture_argument(self, asynchronous):native = self.native(asynchronous)- xhr = self.native(asynchronous, body, {"content-type": content_type})-- response = await self.convert(native, asynchronous, max_body_bytes=4, xhr_captured=[xhr])-- assert response.body == b"data"- assert response.captured_xhr[0].body == body- assert response.captured_xhr[0].headers == {"content-type": content_type}- assert response.captured_xhr[0].meta == {}- xhr.body.assert_called_once_with()+ with pytest.raises(TypeError, match="unexpected keyword argument 'xhr_captured'"):+ await self.convert(native, asynchronous, xhr_captured=[native])+ native.body.assert_not_called()class TestErrorScenarios:diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.pyindex d753d38..3827194 100644--- a/tests/fetchers/test_validator.py+++ b/tests/fetchers/test_validator.py@@ -1,7 +1,10 @@import pytest+from types import SimpleNamespace+from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySessionfrom scrapling.engines._browsers._validators import (validate,+ validate_fetch,StealthConfig,PlaywrightConfig,)@@ -11,17 +14,35 @@ class TestValidators:"""Test configuration validators"""@pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])- @pytest.mark.parametrize("pattern", [None, "", ".*", r"/api/", r"(?i)/api/\d+$"])- def test_capture_xhr_valid(self, model, pattern):- assert validate({"capture_xhr": pattern}, model).capture_xhr == (pattern or None)+ def test_request_recording_replaces_xhr_config(self, model):+ config = validate({"record_requests": True, "max_recorded_requests": 50}, model)+ assert config.record_requests and config.max_recorded_requests == 50+ assert "capture_xhr" not in model.__struct_fields__++ @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])+ @pytest.mark.parametrize("value", [None, ".*"])+ def test_removed_xhr_option_is_rejected_by_config_and_fetch(self, model, value):+ kwargs = {"capture_xhr": value}+ message = "capture_xhr was removed; use record_requests=True and session.network."+ with pytest.raises(TypeError, match=message):+ validate(kwargs, model)+ with pytest.raises(TypeError, match=message):+ validate_fetch(kwargs, SimpleNamespace(_config=model()), model)++ @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])+ def test_other_unknown_options_keep_existing_behavior(self, model):+ config = validate({"unknown_option": True}, model)+ assert not hasattr(config, "unknown_option")+ params = validate_fetch({"unknown_option": True}, SimpleNamespace(_config=config), model)+ assert params.timeout == config.timeout+ assert not hasattr(params, "unknown_option")@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession])- @pytest.mark.parametrize("pattern", ["*/*", "[", "("])- def test_capture_xhr_invalid(self, session_type, pattern):- with pytest.raises(TypeError, match=r"Invalid capture_xhr regex: .*Use '\.\*' to match all URLs"):- session_type(capture_xhr=pattern)+ def test_removed_xhr_option_is_rejected_by_sessions(self, session_type):+ with pytest.raises(TypeError, match="capture_xhr was removed"):+ session_type(capture_xhr=".*")def test_playwright_config_valid(self):"""Test valid PlaywrightConfig"""