Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Take keyword arguments in base parsing methods

Eugenio Lacuesta5982e3477c73
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index b0fb14e24..dd763b607 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -362,12 +362,14 @@ CrawlSpider
This spider also exposes an overrideable method:
- .. method:: parse_start_url(response)
-
- This method is called for the start_urls responses. It allows to parse
- the initial responses and must return either an
- :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`
- object, or an iterable containing any of them.
+ .. method:: parse_start_url(response, **kwargs)
+
+ This method is called for each response produced for the URLs in
+ the spider's ``start_urls`` attribute. It allows to parse
+ the initial responses and must return either an item
+ (:class:`scrapy.item.Item` or :class:`dict`),
+ a :class:`~scrapy.http.Request`,
+ or an iterable containing any of them.
Crawling rules
~~~~~~~~~~~~~~
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 1011eb870..3e19f1e23 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -80,10 +80,10 @@ class Spider(object_ref):
""" This method is deprecated. """
return Request(url, dont_filter=True)
- def _parse(self, response):
- return self.parse(response)
+ def _parse(self, response, **kwargs):
+ return self.parse(response, **kwargs)
- def parse(self, response):
+ def parse(self, response, **kwargs):
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
@classmethod
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index e28d17dcd..4ec0de78c 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -74,10 +74,15 @@ class CrawlSpider(Spider):
super(CrawlSpider, self).__init__(*a, **kw)
self._compile_rules()
- def _parse(self, response):
- return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
+ def _parse(self, response, **kwargs):
+ return self._parse_response(
+ response=response,
+ callback=self.parse_start_url,
+ cb_kwargs=kwargs,
+ follow=True,
+ )
- def parse_start_url(self, response):
+ def parse_start_url(self, response, **kwargs):
return []
def process_results(self, response, results):
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 11bd17db4..4fa6009a5 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
for result_item in self.process_results(response, ret):
yield result_item
- def _parse(self, response):
+ def _parse(self, response, **kwargs):
if not hasattr(self, 'parse_node'):
raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
for result_item in self.process_results(response, ret):
yield result_item
- def _parse(self, response):
+ def _parse(self, response, **kwargs):
if not hasattr(self, 'parse_row'):
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
response = self.adapt_response(response)