Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' of github.com:StasDeep/scrapy into feature/issue-3382
Stas Glubokiy9ab85fe78884
diff --cc docs/topics/contracts.rstindex cac52042a,ba1421c42..ada6fd227--- a/docs/topics/contracts.rst+++ b/docs/topics/contracts.rst@@@ -86,11 -86,8 +86,10 @@@ override three methods.. method:: Contract.adjust_request_args(args)This receives a ``dict`` as an argument containing default arguments- for request object. If ``formdata`` is in ``args``, then- :class:`~scrapy.http.FormRequest` object is created,- otherwise :class:`~scrapy.http.Request` is used.- for :class:`~scrapy.http.Request` object. Must return the same or a- modified version of it.++ for request object. :class:`~scrapy.http.Request` is used++ if ``request_cls`` is not set on ``args``.++ Must return the same or a modified version of it... method:: Contract.pre_process(response)diff --cc scrapy/contracts/__init__.pyindex ca2a8d384,de7ac4503..801c18e73--- a/scrapy/contracts/__init__.py+++ b/scrapy/contracts/__init__.py@@@ -1,9 -1,9 +1,10 @@@import sysimport refrom functools import wraps+ from inspect import getmembersfrom unittest import TestCase+from scrapy import FormRequestfrom scrapy.http import Requestfrom scrapy.utils.spider import iterate_spider_outputfrom scrapy.utils.python import get_spec@@@ -49,20 -49,16 +50,19 @@@ class ContractsManager(object)def from_method(self, method, results):contracts = self.extract_contracts(method)if contracts:-- # calculate request args-- args, kwargs = get_spec(Request.__init__)-- kwargs['callback'] = method++ # prepare request arguments++ kwargs = {'callback': method}for contract in contracts:kwargs = contract.adjust_request_args(kwargs)-- # create and prepare request++ request_cls = kwargs.pop('request_cls', Request)++++ args, _ = get_spec(request_cls.__init__)args.remove('self')++++ # check if all positional arguments are defined in kwargsif set(args).issubset(set(kwargs)):- if 'formdata' in kwargs:- kwargs['method'] = 'POST'- request = FormRequest(**kwargs)- else:- request = Request(**kwargs)- request = Request(**kwargs)++ request = request_cls(**kwargs)# execute pre and post hooks in orderfor contract in reversed(contracts):diff --cc tests/test_contracts.pyindex f2085f711,322d20c4c..c35b068a4--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@@ -25,14 -24,6 +25,15 @@@ class ResponseMock(object)url = 'http://scrapy.org'+class CustomFormContract(Contract):+ name = 'custom_form'++ def adjust_request_args(self, args):++ args['request_cls'] = FormRequest+ args['formdata'] = {'name': 'scrapy'}+ return args++class TestSpider(Spider):name = 'demo_spider'@@@ -109,16 -100,13 +110,20 @@@"""pass+ def custom_form(self, response):+ """+ @url http://scrapy.org+ @custom_form+ """+ pass++ class InheritsTestSpider(TestSpider):+ name = 'inherits_demo_spider'++class ContractsManagerTest(unittest.TestCase):- contracts = [UrlContract, ReturnsContract, ScrapesContract]+ contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract]def setUp(self):self.conman = ContractsManager(self.contracts)@@@ -219,8 -207,8 +224,14 @@@self.assertFalse(self.results.failures)self.assertTrue(self.results.errors)+ def test_form_contract(self):+ spider = TestSpider()+ request = self.conman.from_method(spider.custom_form, self.results)+ self.assertEqual(request.method, 'POST')+ self.assertIsInstance(request, FormRequest)+++ def test_inherited_contracts(self):+ spider = InheritsTestSpider()++ requests = self.conman.from_spider(spider, self.results)+ self.assertTrue(requests)