Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support batches and multiple @url per callback in contracts (#8043)

Adrian46a20dbd6b38
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 144ceb2b3..b18b2916b 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -30,6 +30,24 @@ following example:
@scrapes Title Author Year Price
"""
+A callback docstring can also be split into several batches of contracts, one
+per paragraph separated by a blank line, and a batch may repeat ``@url`` to
+check the callback against more than one sample request:
+
+.. versionadded:: VERSION
+
+.. code-block:: python
+
+ def parse(self, response):
+ """
+ @url http://www.example.com/s?field-keywords=selfish+gene
+ @url http://www.example.com/s?field-keywords=hitchhikers+guide
+ @returns items 16 16
+
+ @url http://www.example.com/s?field-keywords=out+of+stock+item
+ @returns items 0 0
+ """
+
You can use the following contracts:
.. module:: scrapy.contracts.default
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 55fb7e98d..ec0b17ba7 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -78,6 +78,14 @@ class Contract:
"""
request_cls: type[Request] | None = None
+ generates_request: ClassVar[bool] = False
+ """Whether each usage of this contract in a batch produces its own
+ sample request, instead of adjusting a single, shared one.
+
+ :class:`~scrapy.contracts.default.UrlContract` is the only built-in
+ contract that sets this to ``True``, which is what allows a batch to
+ include more than one ``@url`` line.
+ """
name: str
def __init__(self, method: Callable[..., Any], *args: Any):
@@ -183,8 +191,11 @@ class ContractsManager:
return methods
- def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:
- contracts: list[Contract] = []
+ def extract_contracts(self, method: Callable[..., Any]) -> list[list[Contract]]:
+ """Group the contracts of a callback docstring into batches, one per
+ blank line found after the first contract line.
+ """
+ batches: list[list[Contract]] = [[]]
assert method.__doc__ is not None
for line_ in method.__doc__.split("\n"):
line = line_.strip()
@@ -196,16 +207,18 @@ class ContractsManager:
name, args = m.groups()
args = re.split(r"\s+", args)
- contracts.append(self.contracts[name](method, *args))
+ batches[-1].append(self.contracts[name](method, *args))
+ elif not line and batches[-1]:
+ batches.append([])
- return contracts
+ return [batch for batch in batches if batch]
- def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]:
- requests: list[Request | None] = []
+ def from_spider(self, spider: Spider, results: TestResult) -> list[Request]:
+ requests: list[Request] = []
for method in self.tested_methods_from_spidercls(type(spider)):
bound_method = getattr(spider, method)
try:
- requests.append(self.from_method(bound_method, results))
+ requests.extend(self.from_method(bound_method, results))
except Exception:
case = _create_testcase(bound_method, "contract")
results.addError(case, sys.exc_info())
@@ -214,40 +227,63 @@ class ContractsManager:
def from_method(
self, method: Callable[..., Any], results: TestResult
- ) -> Request | None:
- contracts = self.extract_contracts(method)
- if contracts:
- request_cls = Request
- for contract in contracts:
- if contract.request_cls is not None:
- request_cls = contract.request_cls
+ ) -> list[Request]:
+ requests: list[Request] = []
+ for batch in self.extract_contracts(method):
+ requests.extend(self._requests_from_batch(batch, method, results))
+ return requests
+
+ def _requests_from_batch(
+ self,
+ contracts: list[Contract],
+ method: Callable[..., Any],
+ results: TestResult,
+ ) -> list[Request]:
+ request_cls = Request
+ for contract in contracts:
+ if contract.request_cls is not None:
+ request_cls = contract.request_cls
+
+ # calculate request args
+ arg_names, base_kwargs = get_spec(request_cls.__init__)
+ arg_names.remove("self")
+
+ # Don't filter requests to allow
+ # testing different callbacks on the same URL.
+ base_kwargs["dont_filter"] = True
+ base_kwargs["callback"] = method
- # calculate request args
- args, kwargs = get_spec(request_cls.__init__)
+ # Contracts that opt into generating their own request (@url) each
+ # get their own request, built from every other contract in the
+ # batch; other batches only ever build a single, shared request.
+ generators = [contract for contract in contracts if contract.generates_request]
- # Don't filter requests to allow
- # testing different callbacks on the same URL.
- kwargs["dont_filter"] = True
- kwargs["callback"] = method
+ candidates: list[Contract | None] = list(generators) if generators else [None]
+ requests = []
+ for candidate in candidates:
+ kwargs = dict(base_kwargs)
for contract in contracts:
+ if contract in generators and contract is not candidate:
+ continue
kwargs = contract.adjust_request_args(kwargs)
- args.remove("self")
-
# check if all positional arguments are defined in kwargs
- if set(args).issubset(set(kwargs)):
- request = request_cls(**kwargs)
-
- # execute pre and post hooks in order
- for contract in reversed(contracts):
- request = contract.add_pre_hook(request, results)
- for contract in contracts:
- request = contract.add_post_hook(request, results)
-
- self._clean_req(request, method, results)
- return request
- return None
+ if not set(arg_names).issubset(set(kwargs)):
+ continue
+
+ request = request_cls(**kwargs)
+
+ # execute pre and post hooks in order
+ for contract in reversed(contracts):
+ request = contract.add_pre_hook(request, results)
+ for contract in contracts:
+ request = contract.add_post_hook(request, results)
+
+ self._clean_req(request, method, results)
+ requests.append(request)
+
+ return requests
def _clean_req(
self, request: Request, method: Callable[..., Any], results: TestResult
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 38e4f21ce..90d48ec20 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -19,7 +19,8 @@ class UrlContract(Contract):
conditions of a callback.
This contract is mandatory: callbacks lacking it are ignored when running
- the checks.
+ the checks. A callback docstring may repeat it, once per batch, to check
+ the callback against more than one sample URL; see :ref:`topics-contracts`.
.. code-block:: none
@@ -27,6 +28,7 @@ class UrlContract(Contract):
"""
name = "url"
+ generates_request = True
def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
args["url"] = self.args[0]
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 820432987..00b5c463f 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -45,7 +45,7 @@ class CheckSpider(scrapy.Spider):
def parse(self, response, **cb_kwargs):
\"\"\"
@url data:,
- {contracts}
+ {contracts.strip()}
\"\"\"
{parse_def}
""",
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index ea2eed17f..df0419d68 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -33,9 +33,9 @@ if TYPE_CHECKING:
def _request(
conman: ContractsManager, method: CallbackT, results: TestResult
) -> Request:
- request = conman.from_method(method, results)
- assert request
- return request
+ requests = conman.from_method(method, results)
+ assert len(requests) == 1
+ return requests[0]
def _call(request: Request, response: Any) -> Any:
@@ -296,6 +296,18 @@ class DemoSpider(Spider):
key = response.meta["key"]
yield {key: "value"}
+ def returns_multiple_urls(self, response):
+ """checked against two sample urls in one batch, and once more in a
+ second batch, each batch with its own expectations
+ @url http://scrapy.org
+ @url http://example.com
+ @returns items 1 1
+
+ @url http://scrapy.org
+ @returns items 0 0
+ """
+ return DemoItem(url=response.url)
+
class CustomContractSuccessSpider(Spider):
name = "custom_contract_success_spider"
@@ -351,37 +363,38 @@ class TestContractsManager:
def test_contracts(self):
spider = DemoSpider()
- # extract contracts correctly
- contracts = self.conman.extract_contracts(spider.returns_request)
- assert len(contracts) == 2
- assert frozenset(type(x) for x in contracts) == frozenset(
+ # extract contracts correctly, as a single batch
+ batches = self.conman.extract_contracts(spider.returns_request)
+ assert len(batches) == 1
+ assert len(batches[0]) == 2
+ assert frozenset(type(x) for x in batches[0]) == frozenset(
[UrlContract, ReturnsContract]
)
- # returns request for valid method
- assert self.conman.from_method(spider.returns_request, self.results) is not None
+ # returns a request for a valid method
+ assert len(self.conman.from_method(spider.returns_request, self.results)) == 1
# no request for missing url
- assert self.conman.from_method(spider.parse_no_url, self.results) is None
+ assert self.conman.from_method(spider.parse_no_url, self.results) == []
def test_cb_kwargs(self):
spider = DemoSpider()
response = ResponseMock()
# extract contracts correctly
- contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs)
+ (contracts,) = self.conman.extract_contracts(spider.returns_request_cb_kwargs)
assert len(contracts) == 3
assert frozenset(type(x) for x in contracts) == frozenset(
[UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]
)
- contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
+ (contracts,) = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
assert len(contracts) == 3
assert frozenset(type(x) for x in contracts) == frozenset(
[UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]
)
- contracts = self.conman.extract_contracts(
+ (contracts,) = self.conman.extract_contracts(
spider.returns_item_cb_kwargs_error_unexpected_keyword
)
assert len(contracts) == 3
@@ -389,7 +402,7 @@ class TestContractsManager:
[UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]
)
- contracts = self.conman.extract_contracts(
+ (contracts,) = self.conman.extract_contracts(
spider.returns_item_cb_kwargs_error_missing_argument
)
assert len(contracts) == 2
@@ -429,13 +442,13 @@ class TestContractsManager:
spider = DemoSpider()
# extract contracts correctly
- contracts = self.conman.extract_contracts(spider.returns_request_meta)
+ (contracts,) = self.conman.extract_contracts(spider.returns_request_meta)
assert len(contracts) == 3
assert frozenset(type(x) for x in contracts) == frozenset(
[UrlContract, MetadataContract, ReturnsContract]
)
- contracts = self.conman.extract_contracts(spider.returns_item_meta)
+ (contracts,) = self.conman.extract_contracts(spider.returns_item_meta)
assert len(contracts) == 3
assert frozenset(type(x) for x in contracts) == frozenset(
[UrlContract, MetadataContract, ReturnsContract]
@@ -571,6 +584,33 @@ class TestContractsManager:
_call(request, response)
self.should_succeed()
+ def test_multiple_urls_and_batches(self):
+ spider = DemoSpider()
+
+ # a blank line starts a new batch, and each batch may repeat @url
+ batches = self.conman.extract_contracts(spider.returns_multiple_urls)
+ assert len(batches) == 2
+ assert len(batches[0]) == 3
+ assert len(batches[1]) == 2
+
+ requests = self.conman.from_method(spider.returns_multiple_urls, self.results)
+ assert sorted(request.url for request in requests) == [
+ "http://example.com",
+ "http://scrapy.org",
+ "http://scrapy.org",
+ ]
+
+ for request in requests:
+ response = ResponseMock()
+ response.url = request.url
+ _call(request, response)
+
+ # the first batch expects one item from each of its two urls, and
+ # gets it from both; the second batch expects zero items from its
+ # url, but gets one, so only that one fails
+ assert len(self.results.failures) == 1
+ assert not self.results.errors
+
def test_testcase_str(self):
spider = DemoSpider()
contract = UrlContract(spider.returns_request, "http://scrapy.org")
@@ -617,7 +657,7 @@ class TestContractsManager:
response = ResponseMock()
# invalid regex
- assert self.conman.from_method(spider.invalid_regex, self.results) is None
+ assert self.conman.from_method(spider.invalid_regex, self.results) == []
# invalid regex with valid contract
request = _request(