Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
from typing import TYPE_CHECKING, Any
import pytest
from scrapy import Request, Spider
from scrapy.http import JsonRequest, Response
from scrapy.utils.request import request_from_dict
if TYPE_CHECKING:
from twisted.python.failure import Failure
class CustomRequest(Request):
pass
@pytest.fixture
def spider() -> MethodsSpider:
return MethodsSpider()
def _assert_serializes_ok(request: Request, spider: Spider | None = None) -> None:
d = request.to_dict(spider=spider)
request2 = request_from_dict(d, spider=spider)
_assert_same_request(request, request2)
def _assert_same_request(r1: Request, r2: Request) -> None:
assert r1.__class__ == r2.__class__
assert r1.url == r2.url
assert r1.callback == r2.callback
assert r1.errback == r2.errback
assert r1.method == r2.method
assert r1.body == r2.body
assert r1.headers == r2.headers
assert r1.cookies == r2.cookies
assert r1.meta == r2.meta
assert r1.cb_kwargs == r2.cb_kwargs
assert r1.encoding == r2.encoding
assert r1._encoding == r2._encoding
assert r1.priority == r2.priority
assert r1.dont_filter == r2.dont_filter
assert r1.flags == r2.flags
assert r1.id == r2.id
assert r1.parent_id == r2.parent_id
if isinstance(r1, JsonRequest):
assert isinstance(r2, JsonRequest)
assert r1.dumps_kwargs == r2.dumps_kwargs
def test_basic() -> None:
r = Request("http://www.example.com")
_assert_serializes_ok(r)
def test_no_id() -> None:
r = Request("http://www.example.com")
d = r.to_dict()
del d["id"]
assert request_from_dict(d).id != r.id
def test_all_attributes(spider: MethodsSpider) -> None:
r = Request(
url="http://www.example.com",
callback=spider.parse_item,
errback=spider.handle_error,
method="POST",
body=b"some body",
headers={"content-encoding": "text/html; charset=latin-1"},
cookies={"currency": "руб"},
encoding="latin-1",
priority=20,
meta={"a": "b"},
cb_kwargs={"k": "v"},
flags=["testFlag"],
parent_id=1,
)
_assert_serializes_ok(r, spider=spider)
def test_headers() -> None:
r = Request(
"http://www.example.com",
headers=[("X-Multi", "a"), ("x-multi", "b"), ("dnt", "1"), ("Accept", None)],
)
headers = r.to_dict()["headers"]
assert type(headers) is dict
assert headers == {b"X-Multi": [b"a", b"b"], b"dnt": [b"1"], b"Accept": []}
headers[b"dnt"].append(b"2")
assert r.headers.getlist("dnt") == [b"1"]
_assert_serializes_ok(r)
def test_latin1_body() -> None:
r = Request("http://www.example.com", body=b"\xa3")
_assert_serializes_ok(r)
def test_utf8_body() -> None:
r = Request("http://www.example.com", body=b"\xc2\xa3")
_assert_serializes_ok(r)
def test_request_class(spider: MethodsSpider) -> None:
r1 = CustomRequest("http://www.example.com")
_assert_serializes_ok(r1, spider=spider)
r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
_assert_serializes_ok(r2, spider=spider)
def test_callback_serialization(spider: MethodsSpider) -> None:
r = Request(
"http://www.example.com",
callback=spider.parse_item,
errback=spider.handle_error,
)
_assert_serializes_ok(r, spider=spider)
def test_reference_callback_serialization(spider: MethodsSpider) -> None:
r = Request(
"http://www.example.com",
callback=spider.parse_item_reference, # type: ignore[arg-type,misc]
errback=spider.handle_error_reference, # type: ignore[arg-type,misc]
)
_assert_serializes_ok(r, spider=spider)
request_dict = r.to_dict(spider=spider)
assert request_dict["callback"] == "parse_item_reference"
assert request_dict["errback"] == "handle_error_reference"
def test_private_reference_callback_serialization(spider: MethodsSpider) -> None:
r = Request(
"http://www.example.com",
callback=spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]
errback=spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]
)
_assert_serializes_ok(r, spider=spider)
request_dict = r.to_dict(spider=spider)
assert request_dict["callback"] == "_MethodsSpider__parse_item_reference"
assert request_dict["errback"] == "_MethodsSpider__handle_error_reference"
def test_private_callback_serialization(spider: MethodsSpider) -> None:
r = Request(
"http://www.example.com",
callback=spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]
errback=spider.handle_error,
)
_assert_serializes_ok(r, spider=spider)
def test_mixin_private_callback_serialization(spider: MethodsSpider) -> None:
r = Request(
"http://www.example.com",
callback=spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]
errback=spider.handle_error,
)
_assert_serializes_ok(r, spider=spider)
def test_delegated_callback_serialization(spider: MethodsSpider) -> None:
r = Request(
"http://www.example.com",
callback=spider.delegated_callback,
errback=spider.handle_error,
)
_assert_serializes_ok(r, spider=spider)
def test_unserializable_callback1(spider: MethodsSpider) -> None:
r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]
with pytest.raises(
ValueError, match="is not an instance method in: <MethodsSpider"
):
r.to_dict(spider=spider)
def test_unserializable_callback2(spider: MethodsSpider) -> None:
r = Request("http://www.example.com", callback=spider.parse_item)
with pytest.raises(ValueError, match="is not an instance method in: None"):
r.to_dict(spider=None)
def test_unserializable_callback3() -> None:
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
name = "my_spider"
def parse(self, response: Response) -> None:
pass
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
spider.parse = None # type: ignore[method-assign,assignment]
with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
r.to_dict(spider=spider)
def test_callback_not_available() -> None:
"""Callback method is not available in the spider passed to from_dict"""
spider = SpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider) # type: ignore[arg-type]
with pytest.raises(
ValueError, match="Method 'delegated_callback' not found in: <Spider"
):
request_from_dict(d, spider=Spider("foo"))
class SpiderMixin:
def __mixin_callback( # pylint: disable=unused-private-member
self, response: Response
) -> None:
pass
class SpiderDelegation:
def delegated_callback(self, response: Response) -> None:
pass
def parse_item(response: Response) -> None:
pass
def handle_error(failure: Failure) -> None:
pass
def private_parse_item(response: Response) -> None:
pass
def private_handle_error(failure: Failure) -> None:
pass
class MethodsSpider(Spider, SpiderMixin):
name = "test"
parse_item_reference = parse_item
handle_error_reference = handle_error
__parse_item_reference = private_parse_item
__handle_error_reference = private_handle_error
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.delegated_callback = SpiderDelegation().delegated_callback
def parse_item(self, response: Response) -> None:
pass
def handle_error(self, failure: Failure) -> None:
pass
def __parse_item_private( # pylint: disable=unused-private-member
self, response: Response
) -> None:
pass