Scrapy, a fast high-level web crawling & scraping framework for Python.
Ignore docstring @ lines that are not registered contracts (#8226)
Adrian36747fc82d16
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex 913949297..76885988a 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -17,7 +17,7 @@ from scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import get_specif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Iteratorfrom twisted.python.failure import Failure@@ -182,36 +182,55 @@ class ContractsManager:)self.contracts[contract.name] = contract- def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]:- is_method = re.compile(r"^\s*@", re.MULTILINE).search- methods = []- for key, value in getmembers(spidercls):- if callable(value) and value.__doc__ and is_method(value.__doc__):- methods.append(key)+ def _iter_contract_batches(self, docstring: str) -> Iterator[list[tuple[str, str]]]:+ """Yield the ``(name, args)`` pairs of the lines of *docstring* that+ declare a registered contract, grouped into batches, one per blank line+ found after the first contract line.- return methods+ Lines that start with ``@`` but do not name a registered contract are+ ignored, so that docstrings may include unrelated content such as+ decorators in code examples.+ """+ batch: list[tuple[str, str]] = []+ for line_ in docstring.split("\n"):+ line = line_.strip()+ if not line:+ if batch:+ yield batch+ batch = []+ continue+ if not line.startswith("@"):+ continue+ m = re.match(r"@(\w+)\s*(.*)", line)+ if m is None:+ continue+ name, args = m.groups()+ if name in self.contracts:+ batch.append((name, args))+ if batch:+ yield batch++ def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]:+ return [+ key+ for key, value in getmembers(spidercls)+ if callable(value)+ and value.__doc__+ and any(self._iter_contract_batches(value.__doc__))+ ]def extract_contracts(self, method: Callable[..., Any]) -> list[list[Contract]]:"""Group the contracts of a callback docstring into batches, one perblank line found after the first contract line."""- batches: list[list[Contract]] = [[]]assert method.__doc__ is not None- for line_ in method.__doc__.split("\n"):- line = line_.strip()-- if line.startswith("@"):- m = re.match(r"@(\w+)\s*(.*)", line)- if m is None:- continue- name, args = m.groups()- args = re.split(r"\s+", args)-- batches[-1].append(self.contracts[name](method, *args))- elif not line and batches[-1]:- batches.append([])-- return [batch for batch in batches if batch]+ return [+ [+ self.contracts[name](method, *re.split(r"\s+", args))+ for name, args in batch+ ]+ for batch in self._iter_contract_batches(method.__doc__)+ ]def from_spider(self, spider: Spider, results: TestResult) -> list[Request]:requests: list[Request] = []diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex df0419d68..e1bdc8969 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -331,6 +331,34 @@ class InheritsDemoSpider(DemoSpider):name = "inherits_demo_spider"+class UnregisteredAtLineSpider(Spider):+ """Spider whose docstrings contain ``@`` lines that are not contracts."""++ name = "unregistered_at_line_spider"++ @classmethod+ def update_settings(cls, settings):+ """Docstring with a decorator in a code example:++ .. code-block:: python++ @classmethod+ def update_settings(cls, settings): ...+ """+ super().update_settings(settings)++ def parse(self, response):+ """+ @url http://example.com+ @returns items 1 1++ An unregistered line must not break the registered ones:++ @classmethod+ """+ yield {"name": "test"}++class TestContractsManager:contracts = [UrlContract,@@ -666,6 +694,25 @@ class TestContractsManager:_call(request, response)self.should_succeed()+ def test_unregistered_at_line_is_not_a_tested_method(self):+ # A docstring line starting with @ that does not name a registered+ # contract, e.g. a decorator in a code example, must be ignored.+ tested_methods = self.conman.tested_methods_from_spidercls(+ UnregisteredAtLineSpider+ )+ assert tested_methods == ["parse"]++ def test_unregistered_at_line_is_skipped(self):+ (contracts,) = self.conman.extract_contracts(UnregisteredAtLineSpider().parse)+ assert [type(contract) for contract in contracts] == [+ UrlContract,+ ReturnsContract,+ ]++ def test_unregistered_at_line_does_not_break_checks(self):+ self.conman.from_spider(UnregisteredAtLineSpider(), self.results)+ self.should_succeed()+def test_custom_contracts(self):self.conman.from_spider(CustomContractSuccessSpider(), self.results)self.should_succeed()