Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use autodoc for contracts (#7775)

Adrian41bb09741a5d
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 61aef4bbb..df67bee02 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -30,43 +30,15 @@ You can use the following contracts:
.. module:: scrapy.contracts.default
-.. class:: UrlContract
+.. autoclass:: UrlContract
- This contract (``@url``) sets the sample URL used when checking other
- contract conditions for this spider. This contract is mandatory. All
- callbacks lacking this contract are ignored when running the checks::
+.. autoclass:: CallbackKeywordArgumentsContract
- @url url
+.. autoclass:: MetadataContract
-.. class:: CallbackKeywordArgumentsContract
+.. autoclass:: ReturnsContract
- This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
- attribute for the sample request. It must be a valid JSON dictionary.
- ::
-
- @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: MetadataContract
-
- This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`
- attribute for the sample request. It must be a valid JSON dictionary.
- ::
-
- @meta {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: ReturnsContract
-
- This contract (``@returns``) sets lower and upper bounds for the items and
- requests returned by the spider. The upper bound is optional::
-
- @returns item(s)|request(s) [min [max]]
-
-.. class:: ScrapesContract
-
- This contract (``@scrapes``) checks that all the items returned by the
- callback have the specified fields::
-
- @scrapes field_1 field_2 ...
+.. autoclass:: ScrapesContract
Use the :command:`check` command to run the contract checks.
@@ -89,30 +61,16 @@ override three methods:
.. module:: scrapy.contracts
-.. class:: Contract(method, *args)
-
- :param method: callback function to which the contract is associated
- :type method: collections.abc.Callable
-
- :param args: list of arguments passed into the docstring (whitespace
- separated)
- :type args: list
-
- .. method:: Contract.adjust_request_args(args)
-
- This receives a ``dict`` as an argument containing default arguments
- for request object. :class:`~scrapy.Request` is used by default,
- but this can be changed with the ``request_cls`` attribute.
- If multiple contracts in chain have this attribute defined, the last one is used.
+.. autoclass:: Contract
- Must return the same or a modified version of it.
+ .. automethod:: adjust_request_args
- .. method:: Contract.pre_process(response)
+ .. method:: pre_process(response)
This allows hooking in various checks on the response received from the
sample request, before it's being passed to the callback.
- .. method:: Contract.post_process(output)
+ .. method:: post_process(output)
This allows processing the output of the callback. Iterators are
converted to lists before being passed to this hook.
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index ebbdf1b98..dbdbbd456 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -22,7 +22,16 @@ if TYPE_CHECKING:
class Contract:
- """Abstract class for contracts"""
+ """Base class for :ref:`custom contracts <topics-contracts>`.
+
+ *method* is the callback function to which the contract is associated.
+
+ *args* is the list of arguments passed into the docstring, separated by
+ whitespace.
+
+ Subclasses may override :meth:`adjust_request_args`, and define a
+ ``pre_process`` method or a ``post_process`` method, or both.
+ """
request_cls: type[Request] | None = None
name: str
@@ -90,6 +99,13 @@ class Contract:
return request
def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+ """Receive a ``dict`` with the default arguments for the sample request
+ and return it, either unmodified or with changes.
+
+ :class:`~scrapy.Request` is used by default, but this can be changed
+ with the ``request_cls`` attribute. If multiple contracts in the chain
+ define this attribute, the last one is used.
+ """
return args
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 9b42ca36f..e2e27165a 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -15,8 +15,15 @@ if TYPE_CHECKING:
# contracts
class UrlContract(Contract):
- """Contract to set the url of the request (mandatory)
- @url http://scrapy.org
+ """Sets (``@url``) the sample URL used when checking the other contract
+ conditions of a callback.
+
+ This contract is mandatory: callbacks lacking it are ignored when running
+ the checks.
+
+ .. code-block:: none
+
+ @url url
"""
name = "url"
@@ -27,10 +34,14 @@ class UrlContract(Contract):
class CallbackKeywordArgumentsContract(Contract):
- """Contract to set the keyword arguments for the request.
- The value should be a JSON-encoded dictionary, e.g.:
+ """Sets (``@cb_kwargs``) the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
+ attribute of the sample request.
+
+ Its value must be a valid JSON dictionary.
- @cb_kwargs {"arg1": "some value"}
+ .. code-block:: none
+
+ @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
"""
name = "cb_kwargs"
@@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract):
class MetadataContract(Contract):
- """Contract to set metadata arguments for the request.
- The value should be JSON-encoded dictionary, e.g.:
+ """Sets (``@meta``) the :attr:`meta <scrapy.Request.meta>` attribute of the
+ sample request.
+
+ Its value must be a valid JSON dictionary.
- @meta {"arg1": "some value"}
+ .. code-block:: none
+
+ @meta {"arg1": "value1", "arg2": "value2", ...}
"""
name = "meta"
@@ -55,16 +70,29 @@ class MetadataContract(Contract):
class ReturnsContract(Contract):
- """Contract to check the output of a callback
+ """Sets (``@returns``) lower and upper bounds for the items and requests
+ returned by a callback.
+
+ The upper bound is optional:
- general form:
- @returns request(s)/item(s) [min=1 [max]]
+ .. code-block:: none
- e.g.:
- @returns request
- @returns request 2
- @returns request 2 10
- @returns request 0 10
+ @returns item(s)|request(s) [min [max]]
+
+ For example:
+
+ .. code-block:: none
+
+ @returns request
+ @returns request 2
+ @returns request 2 10
+ @returns request 0 10
+
+ Set both bounds to the same value to require an exact number:
+
+ .. code-block:: none
+
+ @returns request 2 2
"""
name = "returns"
@@ -115,8 +143,12 @@ class ReturnsContract(Contract):
class ScrapesContract(Contract):
- """Contract to check presence of fields in scraped items
- @scrapes page_name page_body
+ """Checks (``@scrapes``) that all items returned by a callback have the
+ specified fields.
+
+ .. code-block:: none
+
+ @scrapes field_1 field_2 ...
"""
name = "scrapes"