Scrapy, a fast high-level web crawling & scraping framework for Python.
Add support for meta in Spider Contracts
Laerte Pereirab85e5a66ede0
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rstindex 2d61026e9..a912ff986 100644--- a/docs/topics/contracts.rst+++ b/docs/topics/contracts.rst@@ -46,6 +46,14 @@ This callback is tested using three built-in contracts:@cb_kwargs {"arg1": "value1", "arg2": "value2", ...}+.. class:: MetadataContract++ This contract (``@meta``) sets the :attr:` meta <scrapy.Request.meta>`+ attribute for the sample request. It must be a valid JSON dictionary.+ ::++ @meta {"arg1": "value1", "arg2": "value2", ...}+.. class:: ReturnsContractThis contract (``@returns``) sets lower and upper bounds for the items anddiff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.pyindex 71ca4168a..87099b950 100644--- a/scrapy/contracts/default.py+++ b/scrapy/contracts/default.py@@ -35,6 +35,20 @@ class CallbackKeywordArgumentsContract(Contract):return args+class MetadataContract(Contract):+ """Contract to key metadata arguments for the request.+ The value should be JSON-encoded dictionary, e.g.:++ @meta {"arg1": "some value"}+ """++ name = "meta"++ def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:+ args["meta"] = json.loads(" ".join(self.args))+ return args++class ReturnsContract(Contract):"""Contract to check the output of a callbackdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 932475fb5..7ba0128a5 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -333,6 +333,7 @@ SPIDER_CONTRACTS = {}SPIDER_CONTRACTS_BASE = {"scrapy.contracts.default.UrlContract": 1,"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,+ "scrapy.contracts.default.MetadataContract": 1,"scrapy.contracts.default.ReturnsContract": 2,"scrapy.contracts.default.ScrapesContract": 3,}diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex c9c12f0d8..d578b3af4 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -8,6 +8,7 @@ from scrapy import FormRequestfrom scrapy.contracts import Contract, ContractsManagerfrom scrapy.contracts.default import (CallbackKeywordArgumentsContract,+ MetadataContract,ReturnsContract,ScrapesContract,UrlContract,@@ -29,6 +30,10 @@ class ResponseMock:url = "http://scrapy.org"+class ResponseMetaMock(ResponseMock):+ meta = None++class CustomSuccessContract(Contract):name = "custom_success_contract"@@ -195,6 +200,33 @@ class TestSpider(Spider):"""pass+ def returns_request_meta(self, response):+ """method which returns request+ @url https://example.org+ @meta {"cookiejar": "session1"}+ @returns requests 1+ """+ return Request(+ "https://example.org", meta=response.meta, callback=self.returns_item_meta+ )++ def returns_item_meta(self, response):+ """method which returns item+ @url http://scrapy.org+ @meta {"key": "example"}+ @returns items 1 1+ """+ return TestItem(name="example", url=response.url)++ def returns_error_missing_meta(self, response):+ """method which depends of metadata be defined++ @url http://scrapy.org+ @returns items 1+ """+ key = response.meta["key"]+ yield {key: "value"}+class CustomContractSuccessSpider(Spider):name = "custom_contract_success_spider"@@ -224,6 +256,7 @@ class ContractsManagerTest(unittest.TestCase):contracts = [UrlContract,CallbackKeywordArgumentsContract,+ MetadataContract,ReturnsContract,ScrapesContract,CustomFormContract,@@ -328,6 +361,52 @@ class ContractsManagerTest(unittest.TestCase):request.callback(response, **request.cb_kwargs)self.should_error()+ def test_meta(self):+ spider = TestSpider()++ # extract contracts correctly+ contracts = self.conman.extract_contracts(spider.returns_request_meta)+ self.assertEqual(len(contracts), 3)+ self.assertEqual(+ frozenset(type(x) for x in contracts),+ frozenset([UrlContract, MetadataContract, ReturnsContract]),+ )++ contracts = self.conman.extract_contracts(spider.returns_item_meta)+ self.assertEqual(len(contracts), 3)+ self.assertEqual(+ frozenset(type(x) for x in contracts),+ frozenset([UrlContract, MetadataContract, ReturnsContract]),+ )++ response = ResponseMetaMock()++ # returns_request+ request = self.conman.from_method(spider.returns_request_meta, self.results)+ assert request.meta["cookiejar"] == "session1"+ response.meta = request.meta+ request.callback(response)+ assert response.meta["cookiejar"] == "session1"+ self.should_succeed()++ response = ResponseMetaMock()++ # returns_item+ request = self.conman.from_method(spider.returns_item_meta, self.results)+ assert request.meta["key"] == "example"+ response.meta = request.meta+ request.callback(ResponseMetaMock)+ assert response.meta["key"] == "example"+ self.should_succeed()++ response = ResponseMetaMock()++ request = self.conman.from_method(+ spider.returns_error_missing_meta, self.results+ )+ request.callback(response)+ self.should_error()+def test_returns(self):spider = TestSpider()response = ResponseMock()