Scrapy, a fast high-level web crawling & scraping framework for Python.
Use autodoc for contracts (#7775)
Adrian41bb09741a5d
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rstindex 61aef4bbb..df67bee02 100644--- a/docs/topics/contracts.rst+++ b/docs/topics/contracts.rst@@ -30,43 +30,15 @@ You can use the following contracts:.. module:: scrapy.contracts.default-.. class:: UrlContract+.. autoclass:: UrlContract- This contract (``@url``) sets the sample URL used when checking other- contract conditions for this spider. This contract is mandatory. All- callbacks lacking this contract are ignored when running the checks::+.. autoclass:: CallbackKeywordArgumentsContract- @url url+.. autoclass:: MetadataContract-.. class:: CallbackKeywordArgumentsContract+.. autoclass:: ReturnsContract- This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`- attribute for the sample request. It must be a valid JSON dictionary.- ::-- @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}--.. class:: MetadataContract-- This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`- attribute for the sample request. It must be a valid JSON dictionary.- ::-- @meta {"arg1": "value1", "arg2": "value2", ...}--.. class:: ReturnsContract-- This contract (``@returns``) sets lower and upper bounds for the items and- requests returned by the spider. The upper bound is optional::-- @returns item(s)|request(s) [min [max]]--.. class:: ScrapesContract-- This contract (``@scrapes``) checks that all the items returned by the- callback have the specified fields::-- @scrapes field_1 field_2 ...+.. autoclass:: ScrapesContractUse the :command:`check` command to run the contract checks.@@ -89,30 +61,16 @@ override three methods:.. module:: scrapy.contracts-.. class:: Contract(method, *args)-- :param method: callback function to which the contract is associated- :type method: collections.abc.Callable-- :param args: list of arguments passed into the docstring (whitespace- separated)- :type args: list-- .. method:: Contract.adjust_request_args(args)-- This receives a ``dict`` as an argument containing default arguments- for request object. :class:`~scrapy.Request` is used by default,- but this can be changed with the ``request_cls`` attribute.- If multiple contracts in chain have this attribute defined, the last one is used.+.. autoclass:: Contract- Must return the same or a modified version of it.+ .. automethod:: adjust_request_args- .. method:: Contract.pre_process(response)+ .. method:: pre_process(response)This allows hooking in various checks on the response received from thesample request, before it's being passed to the callback.- .. method:: Contract.post_process(output)+ .. method:: post_process(output)This allows processing the output of the callback. Iterators areconverted to lists before being passed to this hook.diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.pyindex ebbdf1b98..dbdbbd456 100644--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@ -22,7 +22,16 @@ if TYPE_CHECKING:class Contract:- """Abstract class for contracts"""+ """Base class for :ref:`custom contracts <topics-contracts>`.++ *method* is the callback function to which the contract is associated.++ *args* is the list of arguments passed into the docstring, separated by+ whitespace.++ Subclasses may override :meth:`adjust_request_args`, and define a+ ``pre_process`` method or a ``post_process`` method, or both.+ """request_cls: type[Request] | None = Nonename: str@@ -90,6 +99,13 @@ class Contract:return requestdef adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:+ """Receive a ``dict`` with the default arguments for the sample request+ and return it, either unmodified or with changes.++ :class:`~scrapy.Request` is used by default, but this can be changed+ with the ``request_cls`` attribute. If multiple contracts in the chain+ define this attribute, the last one is used.+ """return argsdiff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.pyindex 9b42ca36f..e2e27165a 100644--- a/scrapy/contracts/default.py+++ b/scrapy/contracts/default.py@@ -15,8 +15,15 @@ if TYPE_CHECKING:# contractsclass UrlContract(Contract):- """Contract to set the url of the request (mandatory)- @url http://scrapy.org+ """Sets (``@url``) the sample URL used when checking the other contract+ conditions of a callback.++ This contract is mandatory: callbacks lacking it are ignored when running+ the checks.++ .. code-block:: none++ @url url"""name = "url"@@ -27,10 +34,14 @@ class UrlContract(Contract):class CallbackKeywordArgumentsContract(Contract):- """Contract to set the keyword arguments for the request.- The value should be a JSON-encoded dictionary, e.g.:+ """Sets (``@cb_kwargs``) the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`+ attribute of the sample request.++ Its value must be a valid JSON dictionary.- @cb_kwargs {"arg1": "some value"}+ .. code-block:: none++ @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}"""name = "cb_kwargs"@@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract):class MetadataContract(Contract):- """Contract to set metadata arguments for the request.- The value should be JSON-encoded dictionary, e.g.:+ """Sets (``@meta``) the :attr:`meta <scrapy.Request.meta>` attribute of the+ sample request.++ Its value must be a valid JSON dictionary.- @meta {"arg1": "some value"}+ .. code-block:: none++ @meta {"arg1": "value1", "arg2": "value2", ...}"""name = "meta"@@ -55,16 +70,29 @@ class MetadataContract(Contract):class ReturnsContract(Contract):- """Contract to check the output of a callback+ """Sets (``@returns``) lower and upper bounds for the items and requests+ returned by a callback.++ The upper bound is optional:- general form:- @returns request(s)/item(s) [min=1 [max]]+ .. code-block:: none- e.g.:- @returns request- @returns request 2- @returns request 2 10- @returns request 0 10+ @returns item(s)|request(s) [min [max]]++ For example:++ .. code-block:: none++ @returns request+ @returns request 2+ @returns request 2 10+ @returns request 0 10++ Set both bounds to the same value to require an exact number:++ .. code-block:: none++ @returns request 2 2"""name = "returns"@@ -115,8 +143,12 @@ class ReturnsContract(Contract):class ScrapesContract(Contract):- """Contract to check presence of fields in scraped items- @scrapes page_name page_body+ """Checks (``@scrapes``) that all items returned by a callback have the+ specified fields.++ .. code-block:: none++ @scrapes field_1 field_2 ..."""name = "scrapes"