Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into spider.parse

Eugenio Lacuestabde0384dfda0
diff --cc docs/topics/spiders.rst
index 406f50fb3,89609db7d..bf410c03f
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@@ -417,11 -420,8 +417,13 @@@ Crawling rule
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
instance as first parameter.
+
+.. warning:: Because of its internal implementation, you must explicitly set
+ callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
+ unexpected behaviour can occur otherwise.
+
+ .. versionadded:: 2.0
+ The *errback* parameter.
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
diff --cc scrapy/spiders/__init__.py
index 3e19f1e23,ba1c866f8..586948c24
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@@ -78,12 -78,15 +78,18 @@@ class Spider(object_ref)
def make_requests_from_url(self, url):
""" This method is deprecated. """
+ warnings.warn(
+ "Spider.make_requests_from_url method is deprecated: "
+ "it will be removed and not be called by the default "
+ "Spider.start_requests method in future Scrapy releases. "
+ "Please override Spider.start_requests method instead."
+ )
return Request(url, dont_filter=True)
- def parse(self, response):
+ def _parse(self, response, **kwargs):
+ return self.parse(response, **kwargs)
+
+ def parse(self, response, **kwargs):
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
@classmethod