Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support setting request and spider arguments when checking contracts (#8053)

Adrian3444bdb78305
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 32ae6173e..d88bdb38b 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -341,6 +341,9 @@ check
Run contract checks.
+.. versionadded:: VERSION
+ The ``-a`` option, to pass spider arguments, as in :command:`crawl`.
+
.. skip: start
Usage examples::
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 9272250c0..2b572a768 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -40,6 +40,14 @@ You can use the following contracts:
.. autoclass:: MetadataContract
+.. autoclass:: MethodContract
+
+.. autoclass:: BodyContract
+
+.. autoclass:: HeaderContract
+
+.. autoclass:: CookieContract
+
.. autoclass:: ReturnsContract
.. autoclass:: ScrapesContract
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 0a524daa9..fdb9c7483 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -9,8 +9,9 @@ from unittest import TextTestRunner
from scrapy import Spider
from scrapy.commands import ScrapyCommand
from scrapy.contracts import ContractsManager
+from scrapy.exceptions import UsageError
from scrapy.settings import SETTINGS_PRIORITIES
-from scrapy.utils.conf import build_component_list
+from scrapy.utils.conf import arglist_to_dict, build_component_list
from scrapy.utils.misc import load_object, set_environ
@@ -70,9 +71,23 @@ class Command(ScrapyCommand):
action="store_true",
help="print contract tests for all spiders",
)
+ parser.add_argument(
+ "-a",
+ dest="spargs",
+ action="append",
+ default=[],
+ metavar="NAME=VALUE",
+ help="set spider argument (may be repeated)",
+ )
def process_options(self, args: list[str], opts: argparse.Namespace) -> None:
super().process_options(args, opts)
+ try:
+ opts.spargs = arglist_to_dict(opts.spargs)
+ except ValueError:
+ raise UsageError(
+ "Invalid -a value, use -a NAME=VALUE", print_help=False
+ ) from None
assert self.settings is not None
# Contracts discard the output of callbacks, so item pipelines and
# feed exports get no items, and opening them only causes side
@@ -113,7 +128,7 @@ class Command(ScrapyCommand):
for method in tested_methods:
contract_reqs[spidercls.name].append(method)
elif tested_methods:
- self.crawler_process.crawl(spidercls)
+ self.crawler_process.crawl(spidercls, **opts.spargs)
# start checks
if opts.list:
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index e2e27165a..63553eb0d 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -69,6 +69,84 @@ class MetadataContract(Contract):
return args
+class MethodContract(Contract):
+ """Sets (``@method``) the :attr:`method <scrapy.Request.method>` of the
+ sample request.
+
+ .. versionadded:: VERSION
+
+ .. code-block:: none
+
+ @method POST
+ """
+
+ name = "method"
+
+ def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+ args["method"] = self.args[0]
+ return args
+
+
+class BodyContract(Contract):
+ """Sets (``@body``) the :attr:`body <scrapy.Request.body>` of the sample
+ request.
+
+ .. versionadded:: VERSION
+
+ .. code-block:: none
+
+ @body field1=value1&field2=value2
+ """
+
+ name = "body"
+
+ def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+ args["body"] = " ".join(self.args)
+ return args
+
+
+class HeaderContract(Contract):
+ """Sets (``@header``) a header of the sample request. Use one line per
+ header.
+
+ .. versionadded:: VERSION
+
+ .. code-block:: none
+
+ @header name value
+ """
+
+ name = "header"
+
+ def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+ args["headers"] = {
+ **(args.get("headers") or {}),
+ self.args[0]: " ".join(self.args[1:]),
+ }
+ return args
+
+
+class CookieContract(Contract):
+ """Sets (``@cookie``) a cookie of the sample request. Use one line per
+ cookie.
+
+ .. versionadded:: VERSION
+
+ .. code-block:: none
+
+ @cookie name value
+ """
+
+ name = "cookie"
+
+ def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+ args["cookies"] = {
+ **(args.get("cookies") or {}),
+ self.args[0]: " ".join(self.args[1:]),
+ }
+ return args
+
+
class ReturnsContract(Contract):
"""Sets (``@returns``) lower and upper bounds for the items and requests
returned by a callback.
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 34d071fd5..3461c8943 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -63,7 +63,8 @@ class CookiesMiddleware:
cookie_domain = cookie_domain.removeprefix(".")
hostname = urlparse_cached(request).hostname
- assert hostname is not None
+ if hostname is None:
+ continue
request_domain = hostname.lower()
if cookie_domain and _is_public_domain(cookie_domain):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 0c2444677..15d28fe88 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -561,6 +561,10 @@ SPIDER_CONTRACTS_BASE = {
"scrapy.contracts.default.UrlContract": 1,
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
"scrapy.contracts.default.MetadataContract": 1,
+ "scrapy.contracts.default.MethodContract": 1,
+ "scrapy.contracts.default.BodyContract": 1,
+ "scrapy.contracts.default.HeaderContract": 1,
+ "scrapy.contracts.default.CookieContract": 1,
"scrapy.contracts.default.ReturnsContract": 2,
"scrapy.contracts.default.ScrapesContract": 3,
}
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 17c013338..820432987 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -112,6 +112,42 @@ class CheckSpider(scrapy.Spider):
"""
self._test_contract(proj_path, contracts, parse_def)
+ def test_check_request_contracts(self, proj_path: Path) -> None:
+ contracts = """
+ @method POST
+ @body field1=value1
+ @header X-Header header value
+ @cookie name value
+ """
+ parse_def = """
+ request = response.request
+ if (
+ request.method != 'POST'
+ or request.body != b'field1=value1'
+ or request.headers['X-Header'] != b'header value'
+ or request.cookies != {'name': 'value'}
+ ):
+ raise Exception("Request arguments not set")
+ """
+ self._test_contract(proj_path, contracts, parse_def)
+
+ def test_check_spider_argument(self, proj_path: Path) -> None:
+ parse_def = """
+ if self.key != 'val':
+ raise Exception("Spider argument not set")
+ """
+ self._write_contract(proj_path, "", parse_def)
+ ret, out, err = proc("check", "-a", "key=val", cwd=proj_path)
+ assert "F" not in out
+ assert "OK" in err
+ assert ret == 0
+
+ def test_check_invalid_spider_argument(self, proj_path: Path) -> None:
+ self._write_contract(proj_path, "", "pass")
+ ret, _, err = proc("check", "-a", "FOO", cwd=proj_path)
+ assert ret == 2
+ assert "Invalid -a value, use -a NAME=VALUE" in err
+
def test_check_scrapes_contract(self, proj_path: Path) -> None:
contracts = """
@scrapes key1 key2
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 6c4c80eae..4800e4d29 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -874,3 +874,8 @@ class TestCookiesMiddleware:
cookies2=False,
cookies3=True,
)
+
+ def test_hostless_url(self):
+ request = Request("data:,", cookies={"a": "b"})
+ assert self.mw.process_request(request) is None
+ assert "Cookie" not in request.headers