Scrapy, a fast high-level web crawling & scraping framework for Python.
Support batches and multiple @url per callback in contracts (#8043)
Adrian46a20dbd6b38
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rstindex 144ceb2b3..b18b2916b 100644--- a/docs/topics/contracts.rst+++ b/docs/topics/contracts.rst@@ -30,6 +30,24 @@ following example:@scrapes Title Author Year Price"""+A callback docstring can also be split into several batches of contracts, one+per paragraph separated by a blank line, and a batch may repeat ``@url`` to+check the callback against more than one sample request:++.. versionadded:: VERSION++.. code-block:: python++ def parse(self, response):+ """+ @url http://www.example.com/s?field-keywords=selfish+gene+ @url http://www.example.com/s?field-keywords=hitchhikers+guide+ @returns items 16 16++ @url http://www.example.com/s?field-keywords=out+of+stock+item+ @returns items 0 0+ """+You can use the following contracts:.. module:: scrapy.contracts.defaultdiff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex 55fb7e98d..ec0b17ba7 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -78,6 +78,14 @@ class Contract:"""request_cls: type[Request] | None = None+ generates_request: ClassVar[bool] = False+ """Whether each usage of this contract in a batch produces its own+ sample request, instead of adjusting a single, shared one.++ :class:`~scrapy.contracts.default.UrlContract` is the only built-in+ contract that sets this to ``True``, which is what allows a batch to+ include more than one ``@url`` line.+ """name: strdef __init__(self, method: Callable[..., Any], *args: Any):@@ -183,8 +191,11 @@ class ContractsManager:return methods- def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:- contracts: list[Contract] = []+ def extract_contracts(self, method: Callable[..., Any]) -> list[list[Contract]]:+ """Group the contracts of a callback docstring into batches, one per+ blank line found after the first contract line.+ """+ batches: list[list[Contract]] = [[]]assert method.__doc__ is not Nonefor line_ in method.__doc__.split("\n"):line = line_.strip()@@ -196,16 +207,18 @@ class ContractsManager:name, args = m.groups()args = re.split(r"\s+", args)- contracts.append(self.contracts[name](method, *args))+ batches[-1].append(self.contracts[name](method, *args))+ elif not line and batches[-1]:+ batches.append([])- return contracts+ return [batch for batch in batches if batch]- def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]:- requests: list[Request | None] = []+ def from_spider(self, spider: Spider, results: TestResult) -> list[Request]:+ requests: list[Request] = []for method in self.tested_methods_from_spidercls(type(spider)):bound_method = getattr(spider, method)try:- requests.append(self.from_method(bound_method, results))+ requests.extend(self.from_method(bound_method, results))except Exception:case = _create_testcase(bound_method, "contract")results.addError(case, sys.exc_info())@@ -214,40 +227,63 @@ class ContractsManager:def from_method(self, method: Callable[..., Any], results: TestResult- ) -> Request | None:- contracts = self.extract_contracts(method)- if contracts:- request_cls = Request- for contract in contracts:- if contract.request_cls is not None:- request_cls = contract.request_cls+ ) -> list[Request]:+ requests: list[Request] = []+ for batch in self.extract_contracts(method):+ requests.extend(self._requests_from_batch(batch, method, results))+ return requests++ def _requests_from_batch(+ self,+ contracts: list[Contract],+ method: Callable[..., Any],+ results: TestResult,+ ) -> list[Request]:+ request_cls = Request+ for contract in contracts:+ if contract.request_cls is not None:+ request_cls = contract.request_cls++ # calculate request args+ arg_names, base_kwargs = get_spec(request_cls.__init__)+ arg_names.remove("self")++ # Don't filter requests to allow+ # testing different callbacks on the same URL.+ base_kwargs["dont_filter"] = True+ base_kwargs["callback"] = method- # calculate request args- args, kwargs = get_spec(request_cls.__init__)+ # Contracts that opt into generating their own request (@url) each+ # get their own request, built from every other contract in the+ # batch; other batches only ever build a single, shared request.+ generators = [contract for contract in contracts if contract.generates_request]- # Don't filter requests to allow- # testing different callbacks on the same URL.- kwargs["dont_filter"] = True- kwargs["callback"] = method+ candidates: list[Contract | None] = list(generators) if generators else [None]+ requests = []+ for candidate in candidates:+ kwargs = dict(base_kwargs)for contract in contracts:+ if contract in generators and contract is not candidate:+ continuekwargs = contract.adjust_request_args(kwargs)- args.remove("self")-# check if all positional arguments are defined in kwargs- if set(args).issubset(set(kwargs)):- request = request_cls(**kwargs)-- # execute pre and post hooks in order- for contract in reversed(contracts):- request = contract.add_pre_hook(request, results)- for contract in contracts:- request = contract.add_post_hook(request, results)-- self._clean_req(request, method, results)- return request- return None+ if not set(arg_names).issubset(set(kwargs)):+ continue++ request = request_cls(**kwargs)++ # execute pre and post hooks in order+ for contract in reversed(contracts):+ request = contract.add_pre_hook(request, results)+ for contract in contracts:+ request = contract.add_post_hook(request, results)++ self._clean_req(request, method, results)+ requests.append(request)++ return requestsdef _clean_req(self, request: Request, method: Callable[..., Any], results: TestResultdiff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.pyindex 38e4f21ce..90d48ec20 100644--- a/scrapy/contracts/default.py+++ b/scrapy/contracts/default.py@@ -19,7 +19,8 @@ class UrlContract(Contract):conditions of a callback.This contract is mandatory: callbacks lacking it are ignored when running- the checks.+ the checks. A callback docstring may repeat it, once per batch, to check+ the callback against more than one sample URL; see :ref:`topics-contracts`... code-block:: none@@ -27,6 +28,7 @@ class UrlContract(Contract):"""name = "url"+ generates_request = Truedef adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:args["url"] = self.args[0]diff --git a/tests/test_command_check.py b/tests/test_command_check.pyindex 820432987..00b5c463f 100644--- a/tests/test_command_check.py+++ b/tests/test_command_check.py@@ -45,7 +45,7 @@ class CheckSpider(scrapy.Spider):def parse(self, response, **cb_kwargs):\"\"\"@url data:,- {contracts}+ {contracts.strip()}\"\"\"{parse_def}""",diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex ea2eed17f..df0419d68 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -33,9 +33,9 @@ if TYPE_CHECKING:def _request(conman: ContractsManager, method: CallbackT, results: TestResult) -> Request:- request = conman.from_method(method, results)- assert request- return request+ requests = conman.from_method(method, results)+ assert len(requests) == 1+ return requests[0]def _call(request: Request, response: Any) -> Any:@@ -296,6 +296,18 @@ class DemoSpider(Spider):key = response.meta["key"]yield {key: "value"}+ def returns_multiple_urls(self, response):+ """checked against two sample urls in one batch, and once more in a+ second batch, each batch with its own expectations+ @url http://scrapy.org+ @url http://example.com+ @returns items 1 1++ @url http://scrapy.org+ @returns items 0 0+ """+ return DemoItem(url=response.url)+class CustomContractSuccessSpider(Spider):name = "custom_contract_success_spider"@@ -351,37 +363,38 @@ class TestContractsManager:def test_contracts(self):spider = DemoSpider()- # extract contracts correctly- contracts = self.conman.extract_contracts(spider.returns_request)- assert len(contracts) == 2- assert frozenset(type(x) for x in contracts) == frozenset(+ # extract contracts correctly, as a single batch+ batches = self.conman.extract_contracts(spider.returns_request)+ assert len(batches) == 1+ assert len(batches[0]) == 2+ assert frozenset(type(x) for x in batches[0]) == frozenset([UrlContract, ReturnsContract])- # returns request for valid method- assert self.conman.from_method(spider.returns_request, self.results) is not None+ # returns a request for a valid method+ assert len(self.conman.from_method(spider.returns_request, self.results)) == 1# no request for missing url- assert self.conman.from_method(spider.parse_no_url, self.results) is None+ assert self.conman.from_method(spider.parse_no_url, self.results) == []def test_cb_kwargs(self):spider = DemoSpider()response = ResponseMock()# extract contracts correctly- contracts = self.conman.extract_contracts(spider.returns_request_cb_kwargs)+ (contracts,) = self.conman.extract_contracts(spider.returns_request_cb_kwargs)assert len(contracts) == 3assert frozenset(type(x) for x in contracts) == frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])- contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)+ (contracts,) = self.conman.extract_contracts(spider.returns_item_cb_kwargs)assert len(contracts) == 3assert frozenset(type(x) for x in contracts) == frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])- contracts = self.conman.extract_contracts(+ (contracts,) = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_unexpected_keyword)assert len(contracts) == 3@@ -389,7 +402,7 @@ class TestContractsManager:[UrlContract, CallbackKeywordArgumentsContract, ReturnsContract])- contracts = self.conman.extract_contracts(+ (contracts,) = self.conman.extract_contracts(spider.returns_item_cb_kwargs_error_missing_argument)assert len(contracts) == 2@@ -429,13 +442,13 @@ class TestContractsManager:spider = DemoSpider()# extract contracts correctly- contracts = self.conman.extract_contracts(spider.returns_request_meta)+ (contracts,) = self.conman.extract_contracts(spider.returns_request_meta)assert len(contracts) == 3assert frozenset(type(x) for x in contracts) == frozenset([UrlContract, MetadataContract, ReturnsContract])- contracts = self.conman.extract_contracts(spider.returns_item_meta)+ (contracts,) = self.conman.extract_contracts(spider.returns_item_meta)assert len(contracts) == 3assert frozenset(type(x) for x in contracts) == frozenset([UrlContract, MetadataContract, ReturnsContract]@@ -571,6 +584,33 @@ class TestContractsManager:_call(request, response)self.should_succeed()+ def test_multiple_urls_and_batches(self):+ spider = DemoSpider()++ # a blank line starts a new batch, and each batch may repeat @url+ batches = self.conman.extract_contracts(spider.returns_multiple_urls)+ assert len(batches) == 2+ assert len(batches[0]) == 3+ assert len(batches[1]) == 2++ requests = self.conman.from_method(spider.returns_multiple_urls, self.results)+ assert sorted(request.url for request in requests) == [+ "http://example.com",+ "http://scrapy.org",+ "http://scrapy.org",+ ]++ for request in requests:+ response = ResponseMock()+ response.url = request.url+ _call(request, response)++ # the first batch expects one item from each of its two urls, and+ # gets it from both; the second batch expects zero items from its+ # url, but gets one, so only that one fails+ assert len(self.results.failures) == 1+ assert not self.results.errors+def test_testcase_str(self):spider = DemoSpider()contract = UrlContract(spider.returns_request, "http://scrapy.org")@@ -617,7 +657,7 @@ class TestContractsManager:response = ResponseMock()# invalid regex- assert self.conman.from_method(spider.invalid_regex, self.results) is None+ assert self.conman.from_method(spider.invalid_regex, self.results) == []# invalid regex with valid contractrequest = _request(