Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: adjust the unit tests to the latest changes

Karim shoair49a12d4f8b0f
diff --git a/tests/ai/test_browser_network.py b/tests/ai/test_browser_network.py
index 712a771..adeb0da 100644
--- a/tests/ai/test_browser_network.py
+++ b/tests/ai/test_browser_network.py
@@ -1,7 +1,6 @@
from json import loads
from os import getenv
from asyncio import Event, create_task, sleep, wait_for
-from dataclasses import replace
from types import SimpleNamespace
from urllib.parse import urlsplit
from unittest.mock import AsyncMock, Mock
@@ -13,13 +12,13 @@ from mcp.types import CallToolResult, TextContent
from scrapling.core._types import Any
from scrapling.core._network_formatting import NetworkRequestModel, _request_details
from scrapling.core.ai import NetworkRequestsModel, ScraplingMCPServer, _SessionEntry
-from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest
+from scrapling.engines._browsers._network import NetworkRecorder
from scrapling.engines.toolbelt.custom import Response
def _server() -> tuple[ScraplingMCPServer, Mock]:
server = ScraplingMCPServer()
- saved = Response(
+ record = Response(
url="https://network.test/api/items",
content='{"name":"مرحبا"}'.encode(),
status=201,
@@ -27,17 +26,8 @@ def _server() -> tuple[ScraplingMCPServer, Mock]:
cookies={},
headers={"content-type": "application/json; charset=utf-8", "set-cookie": "test=2"},
request_headers={"content-type": "application/json", "cookie": "test=1"},
- )
- record = NetworkRequest(
- id=4,
- url=saved.url,
method="POST",
- resource_type="fetch",
- status=201,
- request_headers=saved.request_headers,
- request_body=b'{"sent":true}',
- response_headers=saved.headers,
- response=saved,
+ meta={"network_id": 4, "resource_type": "fetch", "request_body": b'{"sent":true}'},
)
network = Mock(spec=["last_id", "dropped_count", "search", "get"], last_id=4, dropped_count=0)
network.search.return_value = [record]
@@ -202,12 +192,16 @@ async def test_network_tools_schema_and_content() -> None:
async def test_network_list_forwards_filters_and_reports_paging_eviction() -> None:
server, network = _server()
network.search.return_value.append(
- replace(
- network.get(4),
- id=8,
+ Response(
url="https://network.test/next",
+ content=b"next",
+ status=200,
+ reason="OK",
+ cookies={},
+ headers={"content-type": "text/plain"},
+ request_headers={},
method="GET",
- resource_type="document",
+ meta={"network_id": 8, "resource_type": "document", "request_body": None},
)
)
network.last_id = 8
@@ -405,7 +399,7 @@ async def test_network_listing_preserves_url_characters_and_only_returns_summary
network._detach()
record = network.get(1)
assert record is not None and record.url == native.url
- assert record.request_body == b'{"secret":"request"}' and record.response.body == b'{"secret":"response"}'
+ assert record.meta["request_body"] == b'{"secret":"request"}' and record.body == b'{"secret":"response"}'
readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
for reader in readers:
reader.assert_awaited_once()
@@ -434,12 +428,13 @@ async def test_network_detail_returns_only_selected_part(part: Any) -> None:
if part == "summary":
assert _detail(result) == _detail_info(4, part, _request_info(4, "/api/items", "POST", status=201))
elif part.endswith("headers"):
- expected = record.response.request_headers if part == "request_headers" else record.response.headers
+ expected = record.request_headers if part == "request_headers" else record.headers
assert _detail(result) == _detail_info(4, part, expected)
else:
expected = '{"sent":true}' if part == "request_body" else '{"name":"مرحبا"}'
assert _detail(result) == _detail_info(4, part, expected)
- assert not hasattr(record, "request")
+ assert isinstance(record, Response) and record.request is None
+ assert not hasattr(record, "response")
network.search.assert_not_called()
@@ -450,7 +445,7 @@ def test_saved_response_details_are_local(part: Any) -> None:
result = _request_details(record, part)
assert isinstance(result, NetworkRequestModel)
if part.endswith("headers"):
- expected = record.response.request_headers if part == "request_headers" else record.response.headers
+ expected = record.request_headers if part == "request_headers" else record.headers
assert result.model_dump() == _detail_info(4, part, expected)
else:
assert result.model_dump() == _detail_info(4, part, '{"name":"مرحبا"}')
@@ -472,7 +467,7 @@ async def test_network_objects_keep_all_summary_fields_and_header_values(part: s
if part == "request_headers":
record.request_headers = headers
elif part == "response_headers":
- record.response_headers = headers
+ record.headers = headers
expected = _request_info(4, "/api/items", "POST", status=201) if part == "summary" else headers
async with Client(server._build_server("127.0.0.1", 8000)) as client:
result = await client.call_tool(
@@ -489,9 +484,9 @@ async def test_network_unicode_body_returns_the_full_original_text(part: str) ->
record = network.get(4)
text = 'Aمرحبا🙂e\u0301\n"z"'
if part == "request_body":
- record.request_body = text.encode()
+ record.meta["request_body"] = text.encode()
else:
- record.response._raw_body = text.encode()
+ record._raw_body = text.encode()
async with Client(server._build_server("127.0.0.1", 8000)) as client:
result = await client.call_tool(
"browser_network_request", {"session_id": "browser", "request_id": 4, "part": part}
@@ -517,7 +512,7 @@ async def test_network_detail_returns_complete_empty_and_large_saved_bodies(part
network._detach()
record = network.get(1)
assert record is not None
- assert (record.request_body if part == "request_body" else record.response.body) == text.encode()
+ assert (record.meta["request_body"] if part == "request_body" else record.body) == text.encode()
readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
for reader in readers:
reader.assert_awaited_once()
@@ -537,14 +532,14 @@ async def test_network_detail_body_availability(case: str) -> None:
server, network = _server()
record = network.get(4)
if case == "empty":
- record.response.status = 204
- record.response._raw_body = b""
+ record.status = 204
+ record._raw_body = b""
elif case == "binary":
- record.response.headers["content-type"] = "image/png"
+ record.headers["content-type"] = "image/png"
elif case == "too_large":
- record.response.meta["body_note"] = "too large; not saved."
+ record.meta["body_note"] = "too large; not saved."
else:
- record.response.meta["body_note"] = "Context closed"
+ record.meta["body_note"] = "Context closed"
result = await server.browser_network_request("browser", 4, "response_body")
expected = {
"empty": _detail_info(4, "response_body", ""),
@@ -561,9 +556,9 @@ async def test_network_detail_body_availability(case: str) -> None:
async def test_network_detail_saved_headers_report_completeness(part: Any, partial: bool) -> None:
server, network = _server()
record = network.get(4)
- headers = record.request_headers if part == "request_headers" else record.response_headers
+ headers = record.request_headers if part == "request_headers" else record.headers
key = "request_headers_partial" if part == "request_headers" else "headers_partial"
- record.response.meta[key] = partial
+ record.meta[key] = partial
result = await server.browser_network_request("browser", 4, part)
assert result.model_dump() == _detail_info(
4, part, headers, note="Headers are partial; full headers unavailable." if partial else None
@@ -588,10 +583,10 @@ async def test_network_detail_decodes_text_for_any_resource(
) -> None:
server, network = _server()
record = network.get(4)
- record.resource_type = "document"
- record.response.headers["content-type"] = content_type
- record.response._raw_body = body
- record.response.encoding = encoding
+ record.meta["resource_type"] = "document"
+ record.headers["content-type"] = content_type
+ record._raw_body = body
+ record.encoding = encoding
result = await server.browser_network_request("browser", 4, "response_body")
assert result.model_dump() == _detail_info(4, "response_body", expected)
@@ -660,9 +655,9 @@ async def test_network_response_uses_saved_encoding_and_request_uses_its_header(
network._detach()
record = network.get(1)
assert record is not None
- assert record.response.encoding == response_encoding
- assert record.response.body == native_body and record.request_body == request_body
- assert record.response_headers == {"content-type": response_type}
+ assert record.encoding == response_encoding
+ assert record.body == native_body and record.meta["request_body"] == request_body
+ assert record.headers == {"content-type": response_type}
assert record.request_headers == {"content-type": request_type}
readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
for reader in readers:
@@ -681,19 +676,18 @@ async def test_network_response_uses_saved_encoding_and_request_uses_its_header(
assert _detail(result) == _detail_info(1, part, {"content-type": expected})
for reader in readers:
reader.assert_awaited_once()
- assert record.response.body == native_body and record.request_body == request_body
+ assert record.body == native_body and record.meta["request_body"] == request_body
@pytest.mark.asyncio
async def test_network_request_body_absent_and_binary() -> None:
server, network = _server()
record = network.get(4)
- record.request_body = None
+ record.meta["request_body"] = None
result = await server.browser_network_request("browser", 4, "request_body")
assert result.model_dump() == _detail_info(4, "request_body", note="Body is absent.")
- record.request_body = b"binary"
+ record.meta["request_body"] = b"binary"
record.request_headers["content-type"] = "application/octet-stream"
- record.response.request_headers["content-type"] = "application/octet-stream"
result = await server.browser_network_request("browser", 4, "request_body")
assert result.model_dump() == _detail_info(4, "request_body", note="Only text bodies can be displayed.")
@@ -737,9 +731,14 @@ async def test_network_reads_saved_parts_without_native_access(
record = network.get(1)
assert record is not None
skipped = part == "response_body" and note == "Non-text body; not saved."
- assert record.response.meta == ({"body_note": note} if skipped else {})
+ assert record.meta == {
+ "network_id": 1,
+ "resource_type": "fetch",
+ "request_body": body if part == "request_body" else None,
+ **({"body_note": note} if skipped else {}),
+ }
if part == "response_body":
- assert record.response.body == (b"" if skipped else body)
+ assert record.body == (b"" if skipped else body)
assert native.existing_response.body.await_count == (not skipped and status != 204)
readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
counts = [reader.await_count for reader in readers]
@@ -759,7 +758,7 @@ async def test_network_reads_saved_parts_without_native_access(
if saved_part == "summary"
else record.request_headers
if saved_part == "request_headers"
- else record.response_headers
+ else record.headers
)
assert _detail(result) == _detail_info(1, saved_part, expected_data)
assert [reader.await_count for reader in readers] == counts
@@ -791,8 +790,13 @@ async def test_network_body_note_does_not_replace_other_saved_parts(error: Excep
await network._contexts[context]["requestfinished"](native)
network._detach()
record = network.get(1)
- assert record is not None and record.response.meta == {"body_note": expected}
- assert record.response.body == b"" and record.request_body == b'{"sent":true}'
+ assert record is not None and record.meta == {
+ "network_id": 1,
+ "resource_type": "fetch",
+ "request_body": b'{"sent":true}',
+ "body_note": expected,
+ }
+ assert record.body == b"" and record.meta["request_body"] == b'{"sent":true}'
assert native.existing_response.body.await_count == (error is not None)
readers = (native.all_headers, native.existing_response.all_headers, native.existing_response.body)
counts = [reader.await_count for reader in readers]
@@ -890,7 +894,7 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
status, content_type, body = responses.get(path, (200, "text/html", html.encode()))
await route.fulfill(status=status, content_type=content_type, body=body, headers={"x-recorded": "yes"})
- async def captured(session: Any, pattern: str) -> NetworkRequest:
+ async def captured(session: Any, pattern: str) -> Response:
while not (records := session.network.search(url_pattern=pattern)):
await sleep(0.01)
return records[0]
@@ -933,8 +937,13 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
}
assert session.network.search(url_pattern="/api/failed$") == []
for name, record in records.items():
- assert record.response.body == (b"" if name == "binary" else responses[f"/api/{name}"][2])
- assert record.response.meta == ({"body_note": "Non-text body; not saved."} if name == "binary" else {})
+ assert record.body == (b"" if name == "binary" else responses[f"/api/{name}"][2])
+ assert record.meta == {
+ "network_id": record.meta["network_id"],
+ "resource_type": "fetch",
+ "request_body": None,
+ **({"body_note": "Non-text body; not saved."} if name == "binary" else {}),
+ }
before = list(seen)
page = session.page_pool.get_ready_page()
assert page is not None
@@ -942,14 +951,14 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
listed = await client.call_tool("browser_network_requests", {"session_id": "browser"})
detail = await client.call_tool(
"browser_network_request",
- {"session_id": "browser", "request_id": request.id, "part": "response_body"},
+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "response_body"},
)
assert not listed.is_error and not detail.is_error
- assert request.response is not None and request.response.body == b'{"items":[1,2]}'
+ assert request is not None and request.body == b'{"items":[1,2]}'
expected_requests = sorted(
- [_request_info(request.id, "/api/items", "POST")]
+ [_request_info(request.meta["network_id"], "/api/items", "POST")]
+ [
- _request_info(record.id, f"/api/{name}", status=responses[f"/api/{name}"][0])
+ _request_info(record.meta["network_id"], f"/api/{name}", status=responses[f"/api/{name}"][0])
for name, record in records.items()
],
key=lambda item: item["id"],
@@ -961,12 +970,14 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
"dropped_count": 0,
}
assert _listing(listed) == expected_listing
- assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')
+ assert _detail(detail) == _detail_info(request.meta["network_id"], "response_body", '{"items":[1,2]}')
request_body = await client.call_tool(
"browser_network_request",
- {"session_id": "browser", "request_id": request.id, "part": "request_body"},
+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "request_body"},
+ )
+ assert _detail(request_body) == _detail_info(
+ request.meta["network_id"], "request_body", '{"sent":true}'
)
- assert _detail(request_body) == _detail_info(request.id, "request_body", '{"sent":true}')
for name, expected, note in (
("empty", "", None),
("absent", "", None),
@@ -975,15 +986,25 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
):
result = await client.call_tool(
"browser_network_request",
- {"session_id": "browser", "request_id": records[name].id, "part": "response_body"},
+ {
+ "session_id": "browser",
+ "request_id": records[name].meta["network_id"],
+ "part": "response_body",
+ },
+ )
+ assert _detail(result) == _detail_info(
+ records[name].meta["network_id"], "response_body", expected, note=note
)
- assert _detail(result) == _detail_info(records[name].id, "response_body", expected, note=note)
headers = await client.call_tool(
"browser_network_request",
- {"session_id": "browser", "request_id": records[name].id, "part": "response_headers"},
+ {
+ "session_id": "browser",
+ "request_id": records[name].meta["network_id"],
+ "part": "response_headers",
+ },
)
assert _detail(headers) == _detail_info(
- records[name].id, "response_headers", records[name].response_headers
+ records[name].meta["network_id"], "response_headers", records[name].headers
)
assert _detail(headers)["data"]["x-recorded"] == "yes"
assert page.state == "busy" and seen == before
@@ -998,15 +1019,16 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
await session.close_pages()
before = list(seen)
detail = await client.call_tool(
- "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}
+ "browser_network_request",
+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "response_body"},
)
- assert _detail(detail) == _detail_info(request.id, "response_body", '{"items":[1,2]}')
+ assert _detail(detail) == _detail_info(request.meta["network_id"], "response_body", '{"items":[1,2]}')
for record in records.values():
saved = await client.call_tool(
"browser_network_request",
- {"session_id": "browser", "request_id": record.id, "part": "response_headers"},
+ {"session_id": "browser", "request_id": record.meta["network_id"], "part": "response_headers"},
)
- assert _detail(saved) == _detail_info(record.id, "response_headers", record.response_headers)
+ assert _detail(saved) == _detail_info(record.meta["network_id"], "response_headers", record.headers)
assert _detail(saved)["data"]["x-recorded"] == "yes"
assert seen == before
finally:
@@ -1015,10 +1037,11 @@ async def test_network_live_capture_actions_navigation_and_reads() -> None:
finally:
await session.close()
assert not session.network._contexts
- assert request.response.body == b'{"items":[1,2]}'
+ assert request.body == b'{"items":[1,2]}'
closed = await client.call_tool("browser_network_requests", {"session_id": "browser"})
assert closed.is_error
closed_detail = await client.call_tool(
- "browser_network_request", {"session_id": "browser", "request_id": request.id, "part": "response_body"}
+ "browser_network_request",
+ {"session_id": "browser", "request_id": request.meta["network_id"], "part": "response_body"},
)
assert closed_detail.is_error
diff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.py
index db77925..edee3f0 100644
--- a/tests/fetchers/test_network_browser.py
+++ b/tests/fetchers/test_network_browser.py
@@ -21,6 +21,7 @@ from playwright.sync_api import Response as PlaywrightResponse
from scrapling.core._types import Any, Generator
from scrapling.core._network_formatting import _request_details
from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
+from scrapling.engines.toolbelt.custom import Response
from scrapling.engines.toolbelt.proxy_rotation import ProxyRotator
@@ -74,6 +75,12 @@ MIME_FETCH = """async paths => {
}
return bodies;
}"""
+SEARCH_FETCH = """async () => {
+ await Promise.all(Array.from({length: 150}, (_, index) => fetch(`/api/search/${index}`, index % 10 ? {
+ method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({index})
+ } : {}).then(response => response.json())));
+ return true;
+}"""
CHARSET_BODY = "caf\u00e9 \u20ac".encode("iso-8859-15")
CHARSET_PATHS = ("/wire/charset", "/wire/charset-spaced", "/wire/raw-charset")
WIRE_PATHS = ["/wire/chunked", "/wire/gzip", "/wire/binary", *CHARSET_PATHS, "/wire/head", "/wire/empty"]
@@ -262,7 +269,7 @@ async def test_async_network_disabled_by_default(session_type: Any) -> None:
def _wait_saved(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:
for _ in range(250):
- records = network.search(url_pattern=pattern, after_id=after_id)
+ records = network.search(url_pattern=pattern, after_id=after_id, limit=None)
if len(records) >= count:
return records
page.wait_for_timeout(20)
@@ -271,44 +278,54 @@ def _wait_saved(page: Any, network: Any, pattern: str, count: int = 1, after_id:
async def _wait_saved_async(page: Any, network: Any, pattern: str, count: int = 1, after_id: int = 0) -> list[Any]:
for _ in range(250):
- records = network.search(url_pattern=pattern, after_id=after_id)
+ records = network.search(url_pattern=pattern, after_id=after_id, limit=None)
if len(records) >= count:
return records
await asyncio.sleep(0.02)
raise AssertionError(f"Expected {count} saved responses for {pattern!r}, got {len(records)}")
-def _check_initial(network: Any, response: Any) -> Any:
+def _check_initial(network: Any) -> Any:
records = network.search()
assert network.enabled
- assert {record.resource_type for record in records} >= {"document", "script", "stylesheet", "image", "fetch", "xhr"}
- assert len({record.id for record in records}) == len(records)
+ assert {record.meta["resource_type"] for record in records} >= {
+ "document",
+ "script",
+ "stylesheet",
+ "image",
+ "fetch",
+ "xhr",
+ }
+ assert len({record.meta["network_id"] for record in records}) == len(records)
initial = network.search(url_pattern=r"/api/initial$", resource_type="fetch")
assert len(initial) == 1
assert initial[0].status == 200
- assert initial[0].request_body is None
- assert initial[0].response.json() == {"ok": True}
- assert "body_note" not in initial[0].response.meta and not hasattr(initial[0], "request")
- assert {urlsplit(item.url).path for item in response.captured_xhr} == {"/api/initial", "/api/xhr"}
- assert all(item.json() == {"ok": True} for item in response.captured_xhr)
+ assert initial[0].meta["request_body"] is None
+ assert initial[0].json() == {"ok": True}
+ assert isinstance(initial[0], Response) and initial[0].request is None
+ assert "body_note" not in initial[0].meta and not hasattr(initial[0], "response")
+ assert network.get(initial[0].meta["network_id"]) is initial[0]
+ captured = network.search(url_pattern=r"/api/(initial|xhr)$")
+ assert {urlsplit(item.url).path for item in captured} == {"/api/initial", "/api/xhr"}
+ assert all(item.json() == {"ok": True} for item in captured)
image = network.search(url_pattern=r"/image\.png$")[0]
- assert image.response.body == b"" and image.response.meta.get("body_note") == "Non-text body; not saved."
+ assert image.body == b"" and image.meta.get("body_note") == "Non-text body; not saved."
for path, body in ((r"/script\.js$", SCRIPT), (r"/style\.css$", b"body { color: black; }")):
asset = network.search(url_pattern=path)[0]
- assert asset.response.body == body and "body_note" not in asset.response.meta
+ assert asset.body == body and "body_note" not in asset.meta
filtered = network.search(include_static=False)
- assert filtered and all(record.resource_type in ("document", "xhr", "fetch") for record in filtered)
+ assert filtered and all(record.meta["resource_type"] in ("document", "xhr", "fetch") for record in filtered)
return initial[0]
def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:
records = network.search(after_id=after_id)
- assert records and all(record.id > after_id for record in records)
+ assert records and all(record.meta["network_id"] > after_id for record in records)
post = network.search(method="POST", status=201)
assert len(post) == 1
- assert loads(post[0].request_body) == {"sent": True}
+ assert loads(post[0].meta["request_body"]) == {"sent": True}
duplicate = network.search(url_pattern=r"/api/repeated$")
- assert len(duplicate) == 2 and duplicate[0].id != duplicate[1].id
+ assert len(duplicate) == 2 and duplicate[0].meta["network_id"] != duplicate[1].meta["network_id"]
assert len(network.search(url_pattern=r"/api/repeated$", limit=1)) == 1
status = network.search(status=503)
assert len(status) == 1
@@ -316,38 +333,34 @@ def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:
redirect = network.search(url_pattern=r"/redirect$")
assert len(redirect) == 1 and redirect[0].status == 302
redirected = network.search(url_pattern=r"/redirected$", status=200)
- assert len(redirected) == 1 and redirected[0].id != redirect[0].id
- assert redirect[0].response_headers["location"] == "/redirected"
+ assert len(redirected) == 1 and redirected[0].meta["network_id"] != redirect[0].meta["network_id"]
+ assert redirect[0].headers["location"] == "/redirected"
return post[0], status[0], network.search(url_pattern=r"/api/large$")[0]
@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])
[email protected]("capture_xhr", [".*", r"/api/"])
-def test_sync_network_browser_history(
- session_type: Any, network_url: str, release_pending: Event, capture_xhr: str
-) -> None:
+def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
with session_type(
**_options(
- capture_xhr=capture_xhr,
page_action=lambda page: page.locator("body[data-initial=ready]").wait_for(state="attached"),
)
) as session:
- response = session.fetch(network_url + "/")
+ session.fetch(network_url + "/")
page = session.page_pool.pages[0].page
_wait_saved(page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6)
- initial = _check_initial(session.network, response)
- assert initial.response.json() == {"ok": True}
+ initial = _check_initial(session.network)
+ assert initial.json() == {"ok": True}
page = session.page_pool.pages[0].page
after_id = session.network.last_id
with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):
page.evaluate(LATER)
_wait_saved(page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7)
post, status, large = _check_later(session.network, after_id)
- assert post.response.request_headers["x-sent"] == "request"
- assert post.response.headers["x-recorded"] == "response"
- assert post.response.json() == {"sent": True}
- assert status.response.body == b"Try later"
- assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""
+ assert post.request_headers["x-sent"] == "request"
+ assert post.headers["x-recorded"] == "response"
+ assert post.json() == {"sent": True}
+ assert status.body == b"Try later"
+ assert large.meta.get("body_note") == "too large; not saved." and large.body == b""
release_pending.clear()
try:
with page.expect_request_finished(lambda request: request.url.endswith("/api/pending")):
@@ -356,7 +369,7 @@ def test_sync_network_browser_history(
assert not session.network.search(url_pattern=r"/api/pending$")
release_pending.set()
pending = _wait_saved(page, session.network, r"/api/pending$")[0]
- assert pending.status == 200 and pending.response.json() == {"ok": True}
+ assert pending.status == 200 and pending.json() == {"ok": True}
finally:
release_pending.set()
other = session.context.new_page()
@@ -370,7 +383,7 @@ def test_sync_network_browser_history(
session.fetch(network_url + "/second")
assert session.page_pool.pages[0].page is page
session.close_pages()
- assert page.is_closed() and session.network.get(post.id).url == post.url
+ assert page.is_closed() and session.network.get(post.meta["network_id"]).url == post.url
fresh_id = session.network.last_id
session.fetch(network_url + "/fresh")
_wait_saved(session.page_pool.pages[0].page, session.network, r"/(?:fresh|api/initial)$", 2, fresh_id)
@@ -383,34 +396,31 @@ def test_sync_network_browser_history(
_wait_saved(page, session.network, r"/api/closing$")
last_id = session.network.last_id
assert session.network.last_id == last_id
- assert session.network.get(post.id).status == 201
- assert session.network.get(post.id).method == "POST"
+ assert session.network.get(post.meta["network_id"]).status == 201
+ assert session.network.get(post.meta["network_id"]).method == "POST"
closing = session.network.search(url_pattern=r"/api/closing$")[0]
- assert closing.response.body == b"closing" and "body_note" not in closing.response.meta
- assert unread.response.json() == {"ok": True}
- assert initial.response.json() == {"ok": True}
- assert post.response.json() == {"sent": True}
- assert post.response.request_headers["x-sent"] == "request"
- assert post.response.headers["x-recorded"] == "response"
+ assert closing.body == b"closing" and "body_note" not in closing.meta
+ assert unread.json() == {"ok": True}
+ assert initial.json() == {"ok": True}
+ assert post.json() == {"sent": True}
+ assert post.request_headers["x-sent"] == "request"
+ assert post.headers["x-recorded"] == "response"
@pytest.mark.asyncio
@pytest.mark.parametrize("session_type", [AsyncDynamicSession, AsyncStealthySession])
[email protected]("capture_xhr", [".*", r"/api/"])
-async def test_async_network_browser_history(
- session_type: Any, network_url: str, release_pending: Event, capture_xhr: str
-) -> None:
+async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
async def initial_page(page: Any) -> None:
await page.locator("body[data-initial=ready]").wait_for(state="attached")
- async with session_type(**_options(capture_xhr=capture_xhr, page_action=initial_page)) as session:
- response = await session.fetch(network_url + "/")
+ async with session_type(**_options(page_action=initial_page)) as session:
+ await session.fetch(network_url + "/")
page = session.page_pool.pages[0].page
await _wait_saved_async(
page, session.network, r"/(?:api/(?:initial|xhr)|script\.js|style\.css|image\.png)?$", 6
)
- initial = _check_initial(session.network, response)
- assert initial.response.json() == {"ok": True}
+ initial = _check_initial(session.network)
+ assert initial.json() == {"ok": True}
page = session.page_pool.pages[0].page
after_id = session.network.last_id
async with page.expect_request_finished(lambda request: request.url.endswith("/api/large")):
@@ -419,11 +429,11 @@ async def test_async_network_browser_history(
page, session.network, r"/(?:api/(?:post|repeated|status|large)|redirect|redirected)$", 7
)
post, status, large = _check_later(session.network, after_id)
- assert post.response.request_headers["x-sent"] == "request"
- assert post.response.headers["x-recorded"] == "response"
- assert post.response.json() == {"sent": True}
- assert status.response.body == b"Try later"
- assert large.response.meta.get("body_note") == "too large; not saved." and large.response.body == b""
+ assert post.request_headers["x-sent"] == "request"
+ assert post.headers["x-recorded"] == "response"
+ assert post.json() == {"sent": True}
+ assert status.body == b"Try later"
+ assert large.meta.get("body_note") == "too large; not saved." and large.body == b""
release_pending.clear()
try:
async with page.expect_request("**/api/pending"):
@@ -431,8 +441,7 @@ async def test_async_network_browser_history(
assert not session.network.search(url_pattern=r"/api/pending$")
release_pending.set()
pending = (await _wait_saved_async(page, session.network, r"/api/pending$"))[0]
- assert pending.status == 200 and pending.response.json() == {"ok": True}
- assert not any("/api/pending" in record.url for record in response.captured_xhr)
+ assert pending.status == 200 and pending.json() == {"ok": True}
finally:
release_pending.set()
other = await session.context.new_page()
@@ -446,7 +455,7 @@ async def test_async_network_browser_history(
await session.fetch(network_url + "/second")
assert session.page_pool.pages[0].page is page
await session.close_pages()
- assert page.is_closed() and session.network.get(post.id).url == post.url
+ assert page.is_closed() and session.network.get(post.meta["network_id"]).url == post.url
fresh_id = session.network.last_id
await session.fetch(network_url + "/fresh")
await _wait_saved_async(
@@ -463,15 +472,15 @@ async def test_async_network_browser_history(
await _wait_saved_async(page, session.network, r"/api/closing$")
last_id = session.network.last_id
assert session.network.last_id == last_id
- assert session.network.get(post.id).status == 201
- assert session.network.get(post.id).method == "POST"
+ assert session.network.get(post.meta["network_id"]).status == 201
+ assert session.network.get(post.meta["network_id"]).method == "POST"
closing = session.network.search(url_pattern=r"/api/closing$")[0]
- assert closing.response.body == b"closing" and "body_note" not in closing.response.meta
- assert unread.response.json() == {"ok": True}
- assert initial.response.json() == {"ok": True}
- assert post.response.json() == {"sent": True}
- assert post.response.request_headers["x-sent"] == "request"
- assert post.response.headers["x-recorded"] == "response"
+ assert closing.body == b"closing" and "body_note" not in closing.meta
+ assert unread.json() == {"ok": True}
+ assert initial.json() == {"ok": True}
+ assert post.json() == {"sent": True}
+ assert post.request_headers["x-sent"] == "request"
+ assert post.headers["x-recorded"] == "response"
def _check_eviction(network: Any) -> int:
@@ -500,8 +509,8 @@ def test_sync_network_browser_eviction_and_clear(session_type: Any, network_url:
page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")
_wait_saved(page, session.network, r"/api/post$")
records = session.network.search()
- assert len(records) == 1 and records[0].id > last_id
- assert records[0].request_body == records[0].response.body == b"fresh"
+ assert len(records) == 1 and records[0].meta["network_id"] > last_id
+ assert records[0].meta["request_body"] == records[0].body == b"fresh"
@pytest.mark.asyncio
@@ -521,8 +530,8 @@ async def test_async_network_browser_eviction_and_clear(session_type: Any, netwo
await page.evaluate("fetch('/api/post', {method: 'POST', body: 'fresh'}).then(response => response.text())")
await _wait_saved_async(page, session.network, r"/api/post$")
records = session.network.search()
- assert len(records) == 1 and records[0].id > last_id
- assert records[0].request_body == records[0].response.body == b"fresh"
+ assert len(records) == 1 and records[0].meta["network_id"] > last_id
+ assert records[0].meta["request_body"] == records[0].body == b"fresh"
PROXY_HTML = '<html><body><script>fetch("/api").then(response => response.json()).then(value => document.body.dataset.api = JSON.stringify(value));</script></body></html>'
@@ -549,10 +558,10 @@ def test_sync_network_browser_temporary_proxy_context(session_type: Any) -> None
session.fetch("https://network.test/")
assert not session.page_pool.pages
record = session.network.search(url_pattern=r"/api$")[0]
- assert record.status == 200 and record.resource_type == "fetch"
- assert record.response.json() == {"temporary": True}
- assert session.network.get(record.id).url == "https://network.test/api"
- assert record.response.json() == {"temporary": True}
+ assert record.status == 200 and record.meta["resource_type"] == "fetch"
+ assert record.json() == {"temporary": True}
+ assert session.network.get(record.meta["network_id"]).url == "https://network.test/api"
+ assert record.json() == {"temporary": True}
@pytest.mark.asyncio
@@ -577,10 +586,10 @@ async def test_async_network_browser_temporary_proxy_context(session_type: Any)
await session.fetch("https://network.test/")
assert not session.page_pool.pages
record = session.network.search(url_pattern=r"/api$")[0]
- assert record.status == 200 and record.resource_type == "fetch"
- assert record.response.json() == {"temporary": True}
- assert session.network.get(record.id).url == "https://network.test/api"
- assert record.response.json() == {"temporary": True}
+ assert record.status == 200 and record.meta["resource_type"] == "fetch"
+ assert record.json() == {"temporary": True}
+ assert session.network.get(record.meta["network_id"]).url == "https://network.test/api"
+ assert record.json() == {"temporary": True}
def _check_saved_override(network: Any, network_url: str) -> None:
@@ -588,9 +597,8 @@ def _check_saved_override(network: Any, network_url: str) -> None:
assert len(records) == 1
record = records[0]
assert record.url == network_url + "/api/overridden"
- assert record.request_body == record.response.body == b"overridden"
+ assert record.meta["request_body"] == record.body == b"overridden"
assert record.request_headers["x-overridden"] == "yes"
- assert record.response.request_headers["x-overridden"] == "yes"
assert not network.search(url_pattern=r"/api/original$")
@@ -670,7 +678,7 @@ def test_sync_network_close_during_unfinished_response(
assert monotonic() - started < 5
assert not gate.release.is_set() and not gate.finished.is_set()
assert page.is_closed()
- assert session.network.get(saved.id).response.body == BLANK
+ assert session.network.get(saved.meta["network_id"]).body == BLANK
assert not session.network.search(url_pattern=r"/api/(?:held|fail)")
assert bool(session.network._contexts) == (target == "page")
finally:
@@ -678,7 +686,7 @@ def test_sync_network_close_during_unfinished_response(
assert gate.finished.wait(5), "The server did not finish the released response"
assert not session.network._contexts
assert session.network.search() == [saved]
- assert saved.response.body == BLANK
+ assert saved.body == BLANK
@pytest.mark.asyncio
@@ -706,7 +714,7 @@ async def test_async_network_close_during_unfinished_response(
await asyncio.wait_for(close(), timeout=5)
assert not gate.release.is_set() and not gate.finished.is_set()
assert page.is_closed()
- assert session.network.get(saved.id).response.body == BLANK
+ assert session.network.get(saved.meta["network_id"]).body == BLANK
assert not session.network.search(url_pattern=r"/api/(?:held|fail)")
assert bool(session.network._contexts) == (target == "page")
finally:
@@ -714,19 +722,19 @@ async def test_async_network_close_during_unfinished_response(
assert await asyncio.to_thread(gate.finished.wait, 5), "The server did not finish the released response"
assert not session.network._contexts
assert session.network.search() == [saved]
- assert saved.response.body == BLANK
+ assert saved.body == BLANK
def _check_burst(network: Any, cursor: int, path: str, expected_paths: list[str]) -> int:
newer = network.search(after_id=cursor, limit=1)
assert len(newer) == 1 and urlsplit(newer[0].url).path == path
- assert newer[0].id == cursor + 1
+ assert newer[0].meta["network_id"] == cursor + 1
records = network.search(limit=2)
- records += network.search(after_id=records[-1].id, limit=2)
+ records += network.search(after_id=records[-1].meta["network_id"], limit=2)
assert [urlsplit(record.url).path for record in records] == expected_paths[-3:]
- assert all(record.response.json() == {"ok": True} for record in records)
+ assert all(record.json() == {"ok": True} for record in records)
assert network.dropped_count == max(0, len(expected_paths) - 3)
- return newer[0].id
+ return newer[0].meta["network_id"]
@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])
@@ -764,7 +772,7 @@ def test_sync_network_burst_completion_order_and_pagination(
assert not session.network._contexts
records = session.network.search()
assert [urlsplit(record.url).path for record in records] == completed[-3:]
- assert all(record.response.json() == {"ok": True} for record in records)
+ assert all(record.json() == {"ok": True} for record in records)
@pytest.mark.asyncio
@@ -805,7 +813,7 @@ async def test_async_network_burst_completion_order_and_pagination(
assert not session.network._contexts
records = session.network.search()
assert [urlsplit(record.url).path for record in records] == completed[-3:]
- assert all(record.response.json() == {"ok": True} for record in records)
+ assert all(record.json() == {"ok": True} for record in records)
def _check_wire_bodies(
@@ -814,29 +822,29 @@ def _check_wire_bodies(
records = {urlsplit(record.url).path: record for record in network.search(url_pattern=r"/wire/")}
assert set(records) == set(WIRE_PATHS)
for path in ("/wire/chunked", "/wire/gzip"):
- assert records[path].response.meta.get("body_note") == "too large; not saved."
- assert records[path].response.body == b""
- assert "content-length" not in records["/wire/chunked"].response_headers
- assert records["/wire/chunked"].response_headers["transfer-encoding"] == "chunked"
- assert int(records["/wire/gzip"].response_headers["content-length"]) < 1024 * 1024
- assert records["/wire/gzip"].response_headers["content-encoding"] == "gzip"
+ assert records[path].meta.get("body_note") == "too large; not saved."
+ assert records[path].body == b""
+ assert "content-length" not in records["/wire/chunked"].headers
+ assert records["/wire/chunked"].headers["transfer-encoding"] == "chunked"
+ assert int(records["/wire/gzip"].headers["content-length"]) < 1024 * 1024
+ assert records["/wire/gzip"].headers["content-encoding"] == "gzip"
assert bytes(wire["/wire/binary"]) == native_bodies["/wire/binary"] == BINARY
for path in ("/wire/binary", "/wire/raw-charset"):
- assert records[path].response.body == b""
- assert records[path].response.meta.get("body_note") == "Non-text body; not saved."
+ assert records[path].body == b""
+ assert records[path].meta.get("body_note") == "Non-text body; not saved."
details = _request_details(records[path], "response_body")
assert details.data is None and details.note == "Non-text body; not saved."
for path in CHARSET_PATHS:
assert bytes(wire[path]) == CHARSET_BODY
- assert records[path].response_headers["content-type"].lower().endswith(("iso-8859-15", '"iso-8859-15"'))
+ assert records[path].headers["content-type"].lower().endswith(("iso-8859-15", '"iso-8859-15"'))
for path, text in native_texts.items():
- assert records[path].response.body == native_bodies[path]
- assert "body_note" not in records[path].response.meta
- assert records[path].response.encoding == "utf-8"
+ assert records[path].body == native_bodies[path]
+ assert "body_note" not in records[path].meta
+ assert records[path].encoding == "utf-8"
assert native_bodies[path].decode("utf-8") == text
- assert records[path].response.get_all_text().strip() == text
+ assert records[path].get_all_text().strip() == text
details = _request_details(records[path], "response_body")
- assert details.data == text and details.request_id == records[path].id
+ assert details.data == text and details.request_id == records[path].meta["network_id"]
assert details.part == "response_body" and details.note is None
assert native_texts["/wire/charset"] == "caf\u00e9 \u20ac"
assert native_bodies["/wire/charset"] == "caf\u00e9 \u20ac".encode("utf-8")
@@ -844,7 +852,7 @@ def _check_wire_bodies(
assert records["/wire/head"].method == "HEAD" and records["/wire/head"].status == 200
assert records["/wire/empty"].status == 204
for path in ("/wire/head", "/wire/empty"):
- assert records[path].response.body == b"" and "body_note" not in records[path].response.meta
+ assert records[path].body == b"" and "body_note" not in records[path].meta
@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])
@@ -891,17 +899,17 @@ def _check_mime_bodies(network: Any, wire: dict[str, list[int]], reads: list[str
for path, (content_type, body) in MIME_RESPONSES.items():
record = records[path]
assert bytes(wire[path]) == body
- assert record.status == 200 and record.method == "GET" and record.resource_type == "fetch"
- assert record.response_headers.get("content-type") == content_type
- assert record.response_headers["x-recorded"] == "response"
- assert int(record.response_headers["content-length"]) == len(body)
+ assert record.status == 200 and record.method == "GET" and record.meta["resource_type"] == "fetch"
+ assert record.headers.get("content-type") == content_type
+ assert record.headers["x-recorded"] == "response"
+ assert int(record.headers["content-length"]) == len(body)
if path in TEXT_RESPONSES:
- assert record.response.body == body and "body_note" not in record.response.meta
+ assert record.body == body and "body_note" not in record.meta
else:
- assert record.response.body == b""
- assert record.response.meta.get("body_note") == "Non-text body; not saved."
+ assert record.body == b""
+ assert record.meta.get("body_note") == "Non-text body; not saved."
for path in ("/mime/json", "/mime/vendor-json"):
- assert records[path].response.json() == {"saved": True}
+ assert records[path].json() == {"saved": True}
@pytest.mark.parametrize(
@@ -956,3 +964,61 @@ async def test_async_network_reads_only_text_bodies(
_check_mime_bodies(session.network, wire, reads)
assert not session.network._contexts
_check_mime_bodies(session.network, wire, reads)
+
+
+def _check_unlimited_search(network: Any, after_id: int) -> None:
+ records = network.search(limit=None)
+ assert len(records) == 125 and network.dropped_count == 25 and network.last_id == after_id + 150
+ assert [record.meta["network_id"] for record in records] == list(range(after_id + 26, after_id + 151))
+ assert network.get(after_id + 25) is None
+ assert network.search() == records[:100]
+ for record in records:
+ index = int(urlsplit(record.url).path.rsplit("/", 1)[-1])
+ assert isinstance(record, Response) and network.get(record.meta["network_id"]) is record
+ assert record.request is None and record.meta["resource_type"] == "fetch"
+ assert record.method == ("POST" if index % 10 else "GET")
+ assert record.status == (201 if index % 10 else 200)
+ assert record.json() == ({"index": index} if index % 10 else {"ok": True})
+ filtered = network.search(
+ url_pattern=r"/api/search/\d+$",
+ method="post",
+ status=201,
+ resource_type="fetch",
+ include_static=False,
+ limit=None,
+ )
+ assert len(filtered) > 100 and filtered == [record for record in records if record.method == "POST"]
+ records.clear()
+ filtered.clear()
+ assert len(network.search(limit=None)) == 125 and len(network.search()) == 100
+
+
[email protected]("session_type", [DynamicSession, StealthySession])
+def test_sync_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:
+ with session_type(**_options(max_recorded_requests=125)) as session:
+ session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ _wait_saved(page, session.network, r"/blank$")
+ after_id = session.network.last_id
+ session.network.clear()
+ assert page.evaluate(SEARCH_FETCH) is True
+ _wait_saved(page, session.network, r"/api/search/", after_id=after_id + 149)
+ _check_unlimited_search(session.network, after_id)
+ assert not session.network._contexts
+ _check_unlimited_search(session.network, after_id)
+
+
[email protected]("session_type", [AsyncDynamicSession, AsyncStealthySession])
+async def test_async_network_unlimited_search_respects_retention(session_type: Any, network_url: str) -> None:
+ async with session_type(**_options(max_recorded_requests=125)) as session:
+ await session.fetch(network_url + "/blank")
+ page = session.page_pool.pages[0].page
+ await _wait_saved_async(page, session.network, r"/blank$")
+ after_id = session.network.last_id
+ session.network.clear()
+ assert await page.evaluate(SEARCH_FETCH) is True
+ await _wait_saved_async(page, session.network, r"/api/search/", after_id=after_id + 149)
+ _check_unlimited_search(session.network, after_id)
+ assert not session.network._contexts
+ _check_unlimited_search(session.network, after_id)
diff --git a/tests/fetchers/test_network_cleanup.py b/tests/fetchers/test_network_cleanup.py
index dc67a75..f7cfc7a 100644
--- a/tests/fetchers/test_network_cleanup.py
+++ b/tests/fetchers/test_network_cleanup.py
@@ -30,8 +30,10 @@ def _closed(session, context, browser, playwright):
def test_sync_session_close_during_capture_does_not_restore_late_response():
session, context, browser, playwright, capture = _session()
+ assert isinstance(session, SyncSession)
capture(request(body=b"saved"))
saved = session.network.get(1)
+ assert saved is not None
def close_before_body_returns():
session.close()
@@ -42,7 +44,7 @@ def test_sync_session_close_during_capture_does_not_restore_late_response():
capture(native)
session.close()
_closed(session, context, browser, playwright)
- assert session.network.search() == [saved] and saved.response.body == b"saved"
+ assert session.network.search() == [saved] and saved.body == b"saved"
assert session.network.last_id == 1
@@ -50,8 +52,10 @@ def test_sync_session_close_during_capture_does_not_restore_late_response():
@pytest.mark.parametrize("read_error", [False, True])
async def test_async_session_close_does_not_wait_for_running_capture(read_error):
session, context, browser, playwright, capture = _session(asynchronous=True)
+ assert isinstance(session, AsyncSession)
await capture(request(body=b"saved", asynchronous=True))
saved = session.network.get(1)
+ assert saved is not None
reading, release = asyncio.Event(), asyncio.Event()
async def held_body():
@@ -75,7 +79,7 @@ async def test_async_session_close_does_not_wait_for_running_capture(read_error)
release.set()
await asyncio.wait_for(task, timeout=2)
await session.close()
- assert session.network.search() == [saved] and saved.response.body == b"saved"
+ assert session.network.search() == [saved] and saved.body == b"saved"
assert session.network.last_id == 1
finally:
release.set()
diff --git a/tests/fetchers/test_network_history.py b/tests/fetchers/test_network_history.py
index 1743b0b..568f4ac 100644
--- a/tests/fetchers/test_network_history.py
+++ b/tests/fetchers/test_network_history.py
@@ -8,7 +8,7 @@ import pytest
from scrapling.core._types import Any, Optional
from scrapling.engines._browsers import _network
-from scrapling.engines._browsers._network import NetworkRecorder, NetworkRequest
+from scrapling.engines._browsers._network import NetworkRecorder
from scrapling.engines.toolbelt.custom import Response
@@ -81,12 +81,14 @@ async def test_only_completed_responses_are_saved(asynchronous):
assert network.search() == []
await finish(context, native)
record = network.get(1)
- assert isinstance(record, NetworkRequest) and not hasattr(record, "request")
+ assert isinstance(record, Response) and record.request is None
assert not hasattr(record, "state") and not hasattr(record, "failure")
- assert record.id == 1 and record.url == native.url and record.method == "GET"
- assert record.resource_type == "fetch" and record.status == 200
- assert record.response.json() == {"ok": True}
- assert "body_note" not in record.response.meta
+ assert not hasattr(record, "response")
+ assert record.meta["network_id"] == 1 and record.url == native.url and record.method == "GET"
+ assert record.meta["resource_type"] == "fetch" and record.status == 200
+ assert record.json() == {"ok": True}
+ assert record.meta == {"network_id": 1, "resource_type": "fetch", "request_body": None}
+ assert network.search()[0] is record and network.get(record.meta["network_id"]) is record
@pytest.mark.asyncio
@@ -104,15 +106,15 @@ async def test_saved_response_has_final_request_data_and_survives_close(asynchro
await finish(context, native)
record = network.get(1)
assert record is not None
- assert record.url == native.url and record.response.url == native.url and record.method == "POST"
- assert record.request_body == b"overridden"
- assert record.request_headers == record.response.request_headers == native.headers
- assert record.response_headers == record.response.headers == native.existing_response.headers
+ assert record.url == native.url and record.method == "POST"
+ assert record.meta["request_body"] == b"overridden"
+ assert record.request_headers == native.headers
+ assert record.headers == native.existing_response.headers
assert network.search(url_pattern=original_url) == [] and network.search(method="POST") == [record]
native.headers["x-overridden"] = "changed"
native.existing_response.headers["content-type"] = "changed"
assert record.request_headers["x-overridden"] == "yes"
- assert record.response_headers["content-type"] == "application/json"
+ assert record.headers["content-type"] == "application/json"
native.all_headers.assert_called_once_with()
native.existing_response.all_headers.assert_called_once_with()
native.existing_response.body.assert_called_once_with()
@@ -121,7 +123,7 @@ async def test_saved_response_has_final_request_data_and_survives_close(asynchro
native.all_headers.side_effect = RuntimeError("Browser closed")
native.existing_response.all_headers.side_effect = RuntimeError("Browser closed")
native.existing_response.body.side_effect = RuntimeError("Browser closed")
- assert record.response.json() == {"ok": True}
+ assert record.json() == {"ok": True}
assert network.get(1) is record and network.search() == [record]
native.existing_response.body.assert_called_once_with()
@@ -145,16 +147,18 @@ async def test_recording_skips_fetch_log_without_muting_normal_responses(monkeyp
await finish(context, native)
record = network.get(1)
assert record is not None
- assert (
- record.response.meta
- == {
+ assert record.meta == {
+ "network_id": 1,
+ "resource_type": "fetch",
+ "request_body": None,
+ **{
"saved": {},
"oversized": {"body_note": "too large; not saved."},
"unreadable": {"body_note": "Body missing"},
"invalid_charset": {},
"non_text": {"body_note": "Non-text body; not saved."},
- }[outcome]
- )
+ }[outcome],
+ }
logger.info.assert_not_called()
response = Response(
url="https://example.test/page",
@@ -178,8 +182,13 @@ async def test_oversized_response_keeps_headers_without_reading_body(monkeypatch
await finish(context, native)
record = network.get(1)
assert record is not None
- assert record.status == 200 and record.response.meta == {"body_note": "too large; not saved."}
- assert record.response.body == b"" and record.response.headers["content-length"] == "5"
+ assert record.status == 200 and record.meta == {
+ "network_id": 1,
+ "resource_type": "fetch",
+ "request_body": None,
+ "body_note": "too large; not saved.",
+ }
+ assert record.body == b"" and record.headers["content-length"] == "5"
native.existing_response.body.assert_not_called()
@@ -194,8 +203,13 @@ async def test_unreadable_response_keeps_headers_and_reports_unavailable(asynchr
record = network.get(1)
assert record is not None
assert record.status == 200
- assert record.response.meta == {"body_note": message or "could not be saved."}
- assert record.response.body == b"" and record.response.headers == native.existing_response.headers
+ assert record.meta == {
+ "network_id": 1,
+ "resource_type": "fetch",
+ "request_body": None,
+ "body_note": message or "could not be saved.",
+ }
+ assert record.body == b"" and record.headers == native.existing_response.headers
@pytest.mark.asyncio
@@ -224,16 +238,21 @@ async def test_non_text_records_keep_metadata_without_using_body_budget(monkeypa
record = network.get(request_id)
assert record is not None
assert record.url == native.url and record.method == "POST" and record.status == 206
- assert record.resource_type == "image" and record.request_body == b"submitted"
- assert record.request_headers == native.headers and record.response_headers == response.headers
- assert record.response.body == b"" and record.response.meta == {"body_note": "Non-text body; not saved."}
+ assert record.meta["resource_type"] == "image" and record.meta["request_body"] == b"submitted"
+ assert record.request_headers == native.headers and record.headers == response.headers
+ assert record.body == b"" and record.meta == {
+ "network_id": request_id,
+ "resource_type": "image",
+ "request_body": b"submitted",
+ "body_note": "Non-text body; not saved.",
+ }
response.body.assert_not_called()
await finish(context, request(body=b"5678", asynchronous=asynchronous))
- assert [record.id for record in network.search()] == [1, 2, 3, 4, 5]
- assert [record.response.body for record in network.search()] == [b"1234", b"", b"", b"", b"5678"]
+ assert [record.meta["network_id"] for record in network.search()] == [1, 2, 3, 4, 5]
+ assert [record.body for record in network.search()] == [b"1234", b"", b"", b"", b"5678"]
assert network.dropped_count == 0
await finish(context, request(body=b"9", asynchronous=asynchronous))
- assert [record.id for record in network.search()] == [2, 3, 4, 5, 6]
+ assert [record.meta["network_id"] for record in network.search()] == [2, 3, 4, 5, 6]
assert network.get(1) is None and network.dropped_count == 1
@@ -244,15 +263,15 @@ async def test_total_body_budget_evicts_oldest_records_and_clear_releases_budget
network, context = recorder(asynchronous=asynchronous)
for body in (b"one", b"two", b"three"):
await finish(context, request(body=body, asynchronous=asynchronous))
- assert [record.id for record in network.search()] == [3]
+ assert [record.meta["network_id"] for record in network.search()] == [3]
record = network.get(3)
assert record is not None
- assert network.dropped_count == 2 and record.response.body == b"three"
+ assert network.dropped_count == 2 and record.body == b"three"
network.clear()
assert network.dropped_count == 0 and network.last_id == 3 and not network.search()
for _ in range(2):
await finish(context, request(body=b"new", asynchronous=asynchronous))
- assert [record.id for record in network.search()] == [4, 5] and network.dropped_count == 0
+ assert [record.meta["network_id"] for record in network.search()] == [4, 5] and network.dropped_count == 0
@pytest.mark.asyncio
@@ -262,13 +281,13 @@ async def test_count_eviction_releases_saved_body_budget(monkeypatch, asynchrono
network, context = recorder(max_requests=1, asynchronous=asynchronous)
for _ in range(3):
await finish(context, request(body=b"new", asynchronous=asynchronous))
- assert [record.id for record in network.search()] == [3]
+ assert [record.meta["network_id"] for record in network.search()] == [3]
record = network.get(3)
assert record is not None
- assert record.response.body == b"new" and network.dropped_count == 2
+ assert record.body == b"new" and network.dropped_count == 2
network.clear()
await finish(context, request(body=b"new", asynchronous=asynchronous))
- assert [record.id for record in network.search()] == [4] and network.dropped_count == 0
+ assert [record.meta["network_id"] for record in network.search()] == [4] and network.dropped_count == 0
@pytest.mark.asyncio
@@ -302,7 +321,7 @@ async def test_late_callback_cannot_publish_after_clear_or_detach(change, starte
await finish(context, request(body=b"new", asynchronous=True))
record = network.get(1)
assert record is not None
- assert record.response.body == b"new"
+ assert record.body == b"new"
finally:
release.set()
await asyncio.gather(*tasks)
@@ -328,10 +347,10 @@ async def test_records_are_published_in_save_order_without_waiting_for_other_bod
await finish(context, request(body=b"second", asynchronous=True))
record = network.get(1)
assert record is not None
- assert record.response.body == b"second"
+ assert record.body == b"second"
release.set()
await asyncio.wait_for(asyncio.gather(*tasks), timeout=2)
- assert [record.response.body for record in network.search()] == [b"second", b"first"]
+ assert [record.body for record in network.search()] == [b"second", b"first"]
finally:
release.set()
await asyncio.gather(*tasks)
@@ -374,13 +393,61 @@ def test_search_filters_include_completed_http_errors():
request(url="https://example.test/api", resource_type="xhr"),
):
context.emit("requestfinished", native)
- assert [record.id for record in network.search(include_static=False)] == [2, 3, 4]
- assert [record.id for record in network.search(after_id=2, limit=1)] == [3]
- assert [record.id for record in network.search(url_pattern="/api$", method="post", status=503)] == [2]
- assert [record.id for record in network.search(resource_type="xhr")] == [4]
+ assert [record.meta["network_id"] for record in network.search(include_static=False)] == [2, 3, 4]
+ assert [record.meta["network_id"] for record in network.search(after_id=2, limit=1)] == [3]
+ assert [record.meta["network_id"] for record in network.search(url_pattern="/api$", method="post", status=503)] == [
+ 2
+ ]
+ assert [record.meta["network_id"] for record in network.search(resource_type="xhr")] == [4]
assert network.get(100) is None and network.search(url_pattern="absent") == []
[email protected]("asynchronous", [False, True])
+async def test_unlimited_search_returns_all_retained_matches_and_an_independent_list(asynchronous):
+ network, context = recorder(max_requests=250, asynchronous=asynchronous)
+ for request_id in range(1, 331):
+ native = request(
+ url=f"https://example.test/{'api' if request_id % 2 == 0 else 'page'}/{request_id}",
+ method="POST" if request_id % 2 == 0 else "GET",
+ resource_type="xhr" if request_id % 2 == 0 else "document",
+ status=503 if request_id % 2 == 0 else 200,
+ body=f'{{"index":{request_id}}}'.encode(),
+ asynchronous=asynchronous,
+ )
+ await finish(context, native)
+ records = network.search(limit=None)
+ assert len(records) == 250 and network.dropped_count == 80 and network.last_id == 330
+ assert [record.meta["network_id"] for record in records] == list(range(81, 331))
+ assert network.get(80) is None
+ assert network.search() == records[:100]
+ assert network.search(limit=7) == records[:7]
+ assert all(isinstance(record, Response) and network.get(record.meta["network_id"]) is record for record in records)
+ assert all(record.request is None and record.json() == {"index": index} for index, record in enumerate(records, 81))
+ filtered = network.search(
+ url_pattern=r"/api/\d+$",
+ method="post",
+ resource_type="xhr",
+ status=503,
+ after_id=100,
+ include_static=False,
+ limit=None,
+ )
+ assert len(filtered) == 115
+ assert [record.meta["network_id"] for record in filtered] == list(range(102, 331, 2))
+ assert network.search(url_pattern="absent", limit=None) == []
+ assert network.search(after_id=330, limit=None) == []
+ records.clear()
+ filtered.reverse()
+ filtered.pop()
+ assert len(network.search(limit=None)) == 250 and network.get(330) is not None
+ assert [record.meta["network_id"] for record in network.search(resource_type="xhr", limit=None)] == list(
+ range(82, 331, 2)
+ )
+ network.clear()
+ assert network.search(limit=None) == [] and network.get(330) is None
+
+
@pytest.mark.parametrize("options", [{"limit": 0}, {"after_id": -1}, {"url_pattern": "["}])
def test_invalid_filters_raise_value_error(options):
with pytest.raises(ValueError):
@@ -428,7 +495,7 @@ async def test_native_handles_are_not_kept_in_history(asynchronous, outcome):
record = network.get(1)
assert record is not None
assert record.status == 200
- assert record.response.meta.get("body_note") == ("Body missing" if outcome == "body_error" else None)
+ assert record.meta.get("body_note") == ("Body missing" if outcome == "body_error" else None)
del native
gc.collect()
assert native_ref() is None and response_ref() is None
@@ -454,7 +521,7 @@ def test_sync_conversion_cannot_restore_history_after_reentrant_reset(change):
native.existing_response.body.side_effect = body
context.emit("requestfinished", native)
expected = [] if change == "detach" else [b"new"]
- assert [record.response.body for record in network.search()] == expected
+ assert [record.body for record in network.search()] == expected
assert network.last_id == len(expected)
@@ -480,7 +547,7 @@ async def test_cancelled_conversion_publishes_nothing_and_later_requests_still_s
await finish(context, request(body=b"saved", asynchronous=True))
record = network.get(1)
assert record is not None
- assert record.response.body == b"saved"
+ assert record.body == b"saved"
finally:
release.set()
await asyncio.gather(*tasks, return_exceptions=True)
@@ -496,10 +563,10 @@ async def test_exact_storage_boundaries_keep_records_until_exceeded(monkeypatch,
network, context = recorder(max_requests=3 if budget == "body" else 2, asynchronous=asynchronous)
for body in (b"1111", b"2222"):
await finish(context, request(body=body, asynchronous=asynchronous))
- assert [record.response.body for record in network.search()] == [b"1111", b"2222"]
+ assert [record.body for record in network.search()] == [b"1111", b"2222"]
assert network.dropped_count == 0
await finish(context, request(body=b"3", asynchronous=asynchronous))
- assert [record.response.body for record in network.search()] == [b"2222", b"3"]
+ assert [record.body for record in network.search()] == [b"2222", b"3"]
assert network.get(1) is None and network.dropped_count == 1
@@ -517,7 +584,7 @@ async def test_unreadable_request_metadata_is_omitted_and_native_handles_are_rel
gc.collect()
assert native_ref() is None and response_ref() is None
await finish(context, request(body=b"next", asynchronous=asynchronous))
- assert [record.response.body for record in network.search()] == [b"next"]
+ assert [record.body for record in network.search()] == [b"next"]
@pytest.mark.asyncio
@@ -552,8 +619,8 @@ async def test_saved_native_bytes_and_text_encoding_remain_valid_after_detach(as
network._detach()
record = network.get(1)
assert record is not None
- assert record.response.body == body
- assert record.response_headers["content-type"] == content_type
- assert record.response.encoding == encoding
- assert record.response.get_all_text().strip() == expected
- assert "body_note" not in record.response.meta
+ assert record.body == body
+ assert record.headers["content-type"] == content_type
+ assert record.encoding == encoding
+ assert record.get_all_text().strip() == expected
+ assert "body_note" not in record.meta
diff --git a/tests/fetchers/test_response_handling.py b/tests/fetchers/test_response_handling.py
index 152d525..aedc2b6 100644
--- a/tests/fetchers/test_response_handling.py
+++ b/tests/fetchers/test_response_handling.py
@@ -3,6 +3,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
from scrapling.parser import Selector
+from scrapling.engines._browsers._base import AsyncSession, SyncSession
from scrapling.engines.toolbelt.convertor import ResponseFactory, Response
@@ -59,6 +60,54 @@ def make_async_page():
class TestResponseFactory:
"""Test ResponseFactory functionality"""
+ @pytest.mark.parametrize("arguments,expected", [({}, "GET"), ({"method": "POST"}, "POST")])
+ def test_response_keeps_request_method(self, arguments, expected):
+ response = Response(
+ url="https://example.com",
+ content=b"saved",
+ status=200,
+ reason="OK",
+ cookies={},
+ headers={},
+ request_headers={},
+ **arguments,
+ )
+
+ assert response.method == expected
+ assert not hasattr(response, "captured_xhr")
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("asynchronous", [False, True])
+ async def test_response_handler_keeps_only_main_frame_navigation(self, asynchronous):
+ main_frame = Mock()
+ page_info = Mock(page=Mock(main_frame=main_frame))
+ responses = [None]
+ latest = None
+ handler = (
+ AsyncSession._create_response_handler(page_info, responses)
+ if asynchronous
+ else SyncSession._create_response_handler(page_info, responses)
+ )
+ for resource_type, navigation, main, captured in (
+ ("document", True, True, True),
+ ("xhr", False, True, False),
+ ("fetch", False, True, False),
+ ("image", False, True, False),
+ ("document", False, True, False),
+ ("document", True, False, False),
+ ("document", True, True, True),
+ ):
+ response = Mock()
+ response.request.resource_type = resource_type
+ response.request.is_navigation_request.return_value = navigation
+ response.request.frame = main_frame if main else Mock()
+ result = handler(response)
+ if result is not None:
+ await result
+ if captured:
+ latest = response
+ assert responses[0] is latest
+
def test_response_from_curl(self):
"""Test creating response from curl_cffi response"""
# Mock curl response
@@ -454,21 +503,11 @@ class TestRecordedResponseFactory:
with pytest.raises(ValueError, match="Failed to get a response"):
await self.convert(None, asynchronous, max_body_bytes=100)
- @pytest.mark.parametrize(
- "content_type,body",
- [("application/json", b'{"saved": true}'), ("application/octet-stream", b"\x00\xffbinary")],
- )
- async def test_recording_rules_do_not_change_captured_xhr(self, asynchronous, content_type, body):
+ async def test_factory_rejects_removed_xhr_capture_argument(self, asynchronous):
native = self.native(asynchronous)
- xhr = self.native(asynchronous, body, {"content-type": content_type})
-
- response = await self.convert(native, asynchronous, max_body_bytes=4, xhr_captured=[xhr])
-
- assert response.body == b"data"
- assert response.captured_xhr[0].body == body
- assert response.captured_xhr[0].headers == {"content-type": content_type}
- assert response.captured_xhr[0].meta == {}
- xhr.body.assert_called_once_with()
+ with pytest.raises(TypeError, match="unexpected keyword argument 'xhr_captured'"):
+ await self.convert(native, asynchronous, xhr_captured=[native])
+ native.body.assert_not_called()
class TestErrorScenarios:
diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.py
index d753d38..3827194 100644
--- a/tests/fetchers/test_validator.py
+++ b/tests/fetchers/test_validator.py
@@ -1,7 +1,10 @@
import pytest
+from types import SimpleNamespace
+
from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
from scrapling.engines._browsers._validators import (
validate,
+ validate_fetch,
StealthConfig,
PlaywrightConfig,
)
@@ -11,17 +14,35 @@ class TestValidators:
"""Test configuration validators"""
@pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
- @pytest.mark.parametrize("pattern", [None, "", ".*", r"/api/", r"(?i)/api/\d+$"])
- def test_capture_xhr_valid(self, model, pattern):
- assert validate({"capture_xhr": pattern}, model).capture_xhr == (pattern or None)
+ def test_request_recording_replaces_xhr_config(self, model):
+ config = validate({"record_requests": True, "max_recorded_requests": 50}, model)
+ assert config.record_requests and config.max_recorded_requests == 50
+ assert "capture_xhr" not in model.__struct_fields__
+
+ @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
+ @pytest.mark.parametrize("value", [None, ".*"])
+ def test_removed_xhr_option_is_rejected_by_config_and_fetch(self, model, value):
+ kwargs = {"capture_xhr": value}
+ message = "capture_xhr was removed; use record_requests=True and session.network."
+ with pytest.raises(TypeError, match=message):
+ validate(kwargs, model)
+ with pytest.raises(TypeError, match=message):
+ validate_fetch(kwargs, SimpleNamespace(_config=model()), model)
+
+ @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
+ def test_other_unknown_options_keep_existing_behavior(self, model):
+ config = validate({"unknown_option": True}, model)
+ assert not hasattr(config, "unknown_option")
+ params = validate_fetch({"unknown_option": True}, SimpleNamespace(_config=config), model)
+ assert params.timeout == config.timeout
+ assert not hasattr(params, "unknown_option")
@pytest.mark.parametrize(
"session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession]
)
- @pytest.mark.parametrize("pattern", ["*/*", "[", "("])
- def test_capture_xhr_invalid(self, session_type, pattern):
- with pytest.raises(TypeError, match=r"Invalid capture_xhr regex: .*Use '\.\*' to match all URLs"):
- session_type(capture_xhr=pattern)
+ def test_removed_xhr_option_is_rejected_by_sessions(self, session_type):
+ with pytest.raises(TypeError, match="capture_xhr was removed"):
+ session_type(capture_xhr=".*")
def test_playwright_config_valid(self):
"""Test valid PlaywrightConfig"""