Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add support for meta in Spider Contracts

Laerte Pereirab85e5a66ede0
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 2d61026e9..a912ff986 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -46,6 +46,14 @@ This callback is tested using three built-in contracts:
@cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
+.. class:: MetadataContract
+
+ This contract (``@meta``) sets the :attr:` meta <scrapy.Request.meta>`
+ attribute for the sample request. It must be a valid JSON dictionary.
+ ::
+
+ @meta {"arg1": "value1", "arg2": "value2", ...}
+
.. class:: ReturnsContract
This contract (``@returns``) sets lower and upper bounds for the items and
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 71ca4168a..87099b950 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract):
return args
+class MetadataContract(Contract):
+ """Contract to key metadata arguments for the request.
+ The value should be JSON-encoded dictionary, e.g.:
+
+ @meta {"arg1": "some value"}
+ """
+
+ name = "meta"
+
+ def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
+ args["meta"] = json.loads(" ".join(self.args))
+ return args
+
+
class ReturnsContract(Contract):
"""Contract to check the output of a callback
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 932475fb5..7ba0128a5 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {}
SPIDER_CONTRACTS_BASE = {
"scrapy.contracts.default.UrlContract": 1,
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
+ "scrapy.contracts.default.MetadataContract": 1,
"scrapy.contracts.default.ReturnsContract": 2,
"scrapy.contracts.default.ScrapesContract": 3,
}
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index c9c12f0d8..d578b3af4 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -8,6 +8,7 @@ from scrapy import FormRequest
from scrapy.contracts import Contract, ContractsManager
from scrapy.contracts.default import (
CallbackKeywordArgumentsContract,
+ MetadataContract,
ReturnsContract,
ScrapesContract,
UrlContract,
@@ -29,6 +30,10 @@ class ResponseMock:
url = "http://scrapy.org"
+class ResponseMetaMock(ResponseMock):
+ meta = None
+
+
class CustomSuccessContract(Contract):
name = "custom_success_contract"
@@ -195,6 +200,33 @@ class TestSpider(Spider):
"""
pass
+ def returns_request_meta(self, response):
+ """method which returns request
+ @url https://example.org
+ @meta {"cookiejar": "session1"}
+ @returns requests 1
+ """
+ return Request(
+ "https://example.org", meta=response.meta, callback=self.returns_item_meta
+ )
+
+ def returns_item_meta(self, response):
+ """method which returns item
+ @url http://scrapy.org
+ @meta {"key": "example"}
+ @returns items 1 1
+ """
+ return TestItem(name="example", url=response.url)
+
+ def returns_error_missing_meta(self, response):
+ """method which depends of metadata be defined
+
+ @url http://scrapy.org
+ @returns items 1
+ """
+ key = response.meta["key"]
+ yield {key: "value"}
+
class CustomContractSuccessSpider(Spider):
name = "custom_contract_success_spider"
@@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase):
contracts = [
UrlContract,
CallbackKeywordArgumentsContract,
+ MetadataContract,
ReturnsContract,
ScrapesContract,
CustomFormContract,
@@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase):
request.callback(response, **request.cb_kwargs)
self.should_error()
+ def test_meta(self):
+ spider = TestSpider()
+
+ # extract contracts correctly
+ contracts = self.conman.extract_contracts(spider.returns_request_meta)
+ self.assertEqual(len(contracts), 3)
+ self.assertEqual(
+ frozenset(type(x) for x in contracts),
+ frozenset([UrlContract, MetadataContract, ReturnsContract]),
+ )
+
+ contracts = self.conman.extract_contracts(spider.returns_item_meta)
+ self.assertEqual(len(contracts), 3)
+ self.assertEqual(
+ frozenset(type(x) for x in contracts),
+ frozenset([UrlContract, MetadataContract, ReturnsContract]),
+ )
+
+ response = ResponseMetaMock()
+
+ # returns_request
+ request = self.conman.from_method(spider.returns_request_meta, self.results)
+ assert request.meta["cookiejar"] == "session1"
+ response.meta = request.meta
+ request.callback(response)
+ assert response.meta["cookiejar"] == "session1"
+ self.should_succeed()
+
+ response = ResponseMetaMock()
+
+ # returns_item
+ request = self.conman.from_method(spider.returns_item_meta, self.results)
+ assert request.meta["key"] == "example"
+ response.meta = request.meta
+ request.callback(ResponseMetaMock)
+ assert response.meta["key"] == "example"
+ self.should_succeed()
+
+ response = ResponseMetaMock()
+
+ request = self.conman.from_method(
+ spider.returns_error_missing_meta, self.results
+ )
+ request.callback(response)
+ self.should_error()
+
def test_returns(self):
spider = TestSpider()
response = ResponseMock()