Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' of github.com:StasDeep/scrapy into feature/issue-3382

Stas Glubokiy9ab85fe78884
diff --cc docs/topics/contracts.rst
index cac52042a,ba1421c42..ada6fd227
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@@ -86,11 -86,8 +86,10 @@@ override three methods
.. method:: Contract.adjust_request_args(args)
This receives a ``dict`` as an argument containing default arguments
- for request object. If ``formdata`` is in ``args``, then
- :class:`~scrapy.http.FormRequest` object is created,
- otherwise :class:`~scrapy.http.Request` is used.
- for :class:`~scrapy.http.Request` object. Must return the same or a
- modified version of it.
++ for request object. :class:`~scrapy.http.Request` is used
++ if ``request_cls`` is not set on ``args``.
+
+ Must return the same or a modified version of it.
.. method:: Contract.pre_process(response)
diff --cc scrapy/contracts/__init__.py
index ca2a8d384,de7ac4503..801c18e73
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@@ -1,9 -1,9 +1,10 @@@
import sys
import re
from functools import wraps
+ from inspect import getmembers
from unittest import TestCase
+from scrapy import FormRequest
from scrapy.http import Request
from scrapy.utils.spider import iterate_spider_output
from scrapy.utils.python import get_spec
@@@ -49,20 -49,16 +50,19 @@@ class ContractsManager(object)
def from_method(self, method, results):
contracts = self.extract_contracts(method)
if contracts:
-- # calculate request args
-- args, kwargs = get_spec(Request.__init__)
-- kwargs['callback'] = method
++ # prepare request arguments
++ kwargs = {'callback': method}
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
-- # create and prepare request
++ request_cls = kwargs.pop('request_cls', Request)
++
++ args, _ = get_spec(request_cls.__init__)
args.remove('self')
++
++ # check if all positional arguments are defined in kwargs
if set(args).issubset(set(kwargs)):
- if 'formdata' in kwargs:
- kwargs['method'] = 'POST'
- request = FormRequest(**kwargs)
- else:
- request = Request(**kwargs)
- request = Request(**kwargs)
++ request = request_cls(**kwargs)
# execute pre and post hooks in order
for contract in reversed(contracts):
diff --cc tests/test_contracts.py
index f2085f711,322d20c4c..c35b068a4
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@@ -25,14 -24,6 +25,15 @@@ class ResponseMock(object)
url = 'http://scrapy.org'
+class CustomFormContract(Contract):
+ name = 'custom_form'
+
+ def adjust_request_args(self, args):
++ args['request_cls'] = FormRequest
+ args['formdata'] = {'name': 'scrapy'}
+ return args
+
+
class TestSpider(Spider):
name = 'demo_spider'
@@@ -109,16 -100,13 +110,20 @@@
"""
pass
+ def custom_form(self, response):
+ """
+ @url http://scrapy.org
+ @custom_form
+ """
+ pass
+
+ class InheritsTestSpider(TestSpider):
+ name = 'inherits_demo_spider'
+
+
class ContractsManagerTest(unittest.TestCase):
- contracts = [UrlContract, ReturnsContract, ScrapesContract]
+ contracts = [UrlContract, ReturnsContract, ScrapesContract, CustomFormContract]
def setUp(self):
self.conman = ContractsManager(self.contracts)
@@@ -219,8 -207,8 +224,14 @@@
self.assertFalse(self.results.failures)
self.assertTrue(self.results.errors)
+ def test_form_contract(self):
+ spider = TestSpider()
+ request = self.conman.from_method(spider.custom_form, self.results)
+ self.assertEqual(request.method, 'POST')
+ self.assertIsInstance(request, FormRequest)
++
+ def test_inherited_contracts(self):
+ spider = InheritsTestSpider()
+
+ requests = self.conman.from_spider(spider, self.results)
+ self.assertTrue(requests)