Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into fix/issue-3380

Daniel Graña2aae514f99d8
diff --cc scrapy/contracts/__init__.py
index 518224225,851a26a8e..f605c9eb1
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@@ -49,14 -49,14 +49,19 @@@ class ContractsManager(object)
def from_method(self, method, results):
contracts = self.extract_contracts(method)
if contracts:
+ request_cls = Request
+ for contract in contracts:
+ if contract.request_cls is not None:
+ request_cls = contract.request_cls
+
# calculate request args
- args, kwargs = get_spec(Request.__init__)
+ args, kwargs = get_spec(request_cls.__init__)
+
+ # Don't filter requests to allow
+ # testing different callbacks on the same URL.
+ kwargs['dont_filter'] = True
kwargs['callback'] = method
+
for contract in contracts:
kwargs = contract.adjust_request_args(kwargs)
diff --cc tests/test_contracts.py
index b4209e1f6,fc5c94771..3571d8f7f
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@@ -5,7 -3,7 +5,8 @@@ from twisted.internet import defe
from twisted.python import failure
from twisted.trial import unittest
+ from scrapy import FormRequest
+from scrapy.crawler import CrawlerRunner
from scrapy.spidermiddlewares.httperror import HttpError
from scrapy.spiders import Spider
from scrapy.http import Request
@@@ -209,38 -224,12 +226,44 @@@ class ContractsManagerTest(unittest.Tes
self.assertFalse(self.results.failures)
self.assertTrue(self.results.errors)
+ @defer.inlineCallbacks
+ def test_same_url(self):
+
+ class TestSameUrlSpider(Spider):
+ name = 'test_same_url'
+
+ def __init__(self, *args, **kwargs):
+ super(TestSameUrlSpider, self).__init__(*args, **kwargs)
+ self.visited = 0
+
+ def start_requests(s):
+ return self.conman.from_spider(s, self.results)
+
+ def parse_first(self, response):
+ self.visited += 1
+ return TestItem()
+
+ def parse_second(self, response):
+ self.visited += 1
+ return TestItem()
+
+ with MockServer() as mockserver:
+ contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
+
+ get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
+ get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
+
+ crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+ yield crawler.crawl()
+
+ self.assertEqual(crawler.spider.visited, 2)
+
+ def test_form_contract(self):
+ spider = TestSpider()
+ request = self.conman.from_method(spider.custom_form, self.results)
+ self.assertEqual(request.method, 'POST')
+ self.assertIsInstance(request, FormRequest)
+
def test_inherited_contracts(self):
spider = InheritsTestSpider()