Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Ignore docstring @ lines that are not registered contracts (#8226)

Adrian36747fc82d16
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 913949297..76885988a 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -17,7 +17,7 @@ from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import get_spec
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Iterator
from twisted.python.failure import Failure
@@ -182,36 +182,55 @@ class ContractsManager:
)
self.contracts[contract.name] = contract
- def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]:
- is_method = re.compile(r"^\s*@", re.MULTILINE).search
- methods = []
- for key, value in getmembers(spidercls):
- if callable(value) and value.__doc__ and is_method(value.__doc__):
- methods.append(key)
+ def _iter_contract_batches(self, docstring: str) -> Iterator[list[tuple[str, str]]]:
+ """Yield the ``(name, args)`` pairs of the lines of *docstring* that
+ declare a registered contract, grouped into batches, one per blank line
+ found after the first contract line.
- return methods
+ Lines that start with ``@`` but do not name a registered contract are
+ ignored, so that docstrings may include unrelated content such as
+ decorators in code examples.
+ """
+ batch: list[tuple[str, str]] = []
+ for line_ in docstring.split("\n"):
+ line = line_.strip()
+ if not line:
+ if batch:
+ yield batch
+ batch = []
+ continue
+ if not line.startswith("@"):
+ continue
+ m = re.match(r"@(\w+)\s*(.*)", line)
+ if m is None:
+ continue
+ name, args = m.groups()
+ if name in self.contracts:
+ batch.append((name, args))
+ if batch:
+ yield batch
+
+ def tested_methods_from_spidercls(self, spidercls: type[Spider]) -> list[str]:
+ return [
+ key
+ for key, value in getmembers(spidercls)
+ if callable(value)
+ and value.__doc__
+ and any(self._iter_contract_batches(value.__doc__))
+ ]
def extract_contracts(self, method: Callable[..., Any]) -> list[list[Contract]]:
"""Group the contracts of a callback docstring into batches, one per
blank line found after the first contract line.
"""
- batches: list[list[Contract]] = [[]]
assert method.__doc__ is not None
- for line_ in method.__doc__.split("\n"):
- line = line_.strip()
-
- if line.startswith("@"):
- m = re.match(r"@(\w+)\s*(.*)", line)
- if m is None:
- continue
- name, args = m.groups()
- args = re.split(r"\s+", args)
-
- batches[-1].append(self.contracts[name](method, *args))
- elif not line and batches[-1]:
- batches.append([])
-
- return [batch for batch in batches if batch]
+ return [
+ [
+ self.contracts[name](method, *re.split(r"\s+", args))
+ for name, args in batch
+ ]
+ for batch in self._iter_contract_batches(method.__doc__)
+ ]
def from_spider(self, spider: Spider, results: TestResult) -> list[Request]:
requests: list[Request] = []
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index df0419d68..e1bdc8969 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -331,6 +331,34 @@ class InheritsDemoSpider(DemoSpider):
name = "inherits_demo_spider"
+class UnregisteredAtLineSpider(Spider):
+ """Spider whose docstrings contain ``@`` lines that are not contracts."""
+
+ name = "unregistered_at_line_spider"
+
+ @classmethod
+ def update_settings(cls, settings):
+ """Docstring with a decorator in a code example:
+
+ .. code-block:: python
+
+ @classmethod
+ def update_settings(cls, settings): ...
+ """
+ super().update_settings(settings)
+
+ def parse(self, response):
+ """
+ @url http://example.com
+ @returns items 1 1
+
+ An unregistered line must not break the registered ones:
+
+ @classmethod
+ """
+ yield {"name": "test"}
+
+
class TestContractsManager:
contracts = [
UrlContract,
@@ -666,6 +694,25 @@ class TestContractsManager:
_call(request, response)
self.should_succeed()
+ def test_unregistered_at_line_is_not_a_tested_method(self):
+ # A docstring line starting with @ that does not name a registered
+ # contract, e.g. a decorator in a code example, must be ignored.
+ tested_methods = self.conman.tested_methods_from_spidercls(
+ UnregisteredAtLineSpider
+ )
+ assert tested_methods == ["parse"]
+
+ def test_unregistered_at_line_is_skipped(self):
+ (contracts,) = self.conman.extract_contracts(UnregisteredAtLineSpider().parse)
+ assert [type(contract) for contract in contracts] == [
+ UrlContract,
+ ReturnsContract,
+ ]
+
+ def test_unregistered_at_line_does_not_break_checks(self):
+ self.conman.from_spider(UnregisteredAtLineSpider(), self.results)
+ self.should_succeed()
+
def test_custom_contracts(self):
self.conman.from_spider(CustomContractSuccessSpider(), self.results)
self.should_succeed()