Scrapy, a fast high-level web crawling & scraping framework for Python.
Take keyword arguments in base parsing methods
Eugenio Lacuesta5982e3477c73
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex b0fb14e24..dd763b607 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -362,12 +362,14 @@ CrawlSpiderThis spider also exposes an overrideable method:- .. method:: parse_start_url(response)-- This method is called for the start_urls responses. It allows to parse- the initial responses and must return either an- :class:`~scrapy.item.Item` object, a :class:`~scrapy.http.Request`- object, or an iterable containing any of them.+ .. method:: parse_start_url(response, **kwargs)++ This method is called for each response produced for the URLs in+ the spider's ``start_urls`` attribute. It allows to parse+ the initial responses and must return either an item+ (:class:`scrapy.item.Item` or :class:`dict`),+ a :class:`~scrapy.http.Request`,+ or an iterable containing any of them.Crawling rules~~~~~~~~~~~~~~diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.pyindex 1011eb870..3e19f1e23 100644--- a/scrapy/spiders/__init__.py+++ b/scrapy/spiders/__init__.py@@ -80,10 +80,10 @@ class Spider(object_ref):""" This method is deprecated. """return Request(url, dont_filter=True)- def _parse(self, response):- return self.parse(response)+ def _parse(self, response, **kwargs):+ return self.parse(response, **kwargs)- def parse(self, response):+ def parse(self, response, **kwargs):raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))@classmethoddiff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.pyindex e28d17dcd..4ec0de78c 100644--- a/scrapy/spiders/crawl.py+++ b/scrapy/spiders/crawl.py@@ -74,10 +74,15 @@ class CrawlSpider(Spider):super(CrawlSpider, self).__init__(*a, **kw)self._compile_rules()- def _parse(self, response):- return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)+ def _parse(self, response, **kwargs):+ return self._parse_response(+ response=response,+ callback=self.parse_start_url,+ cb_kwargs=kwargs,+ follow=True,+ )- def parse_start_url(self, response):+ def parse_start_url(self, response, **kwargs):return []def process_results(self, response, results):diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.pyindex 11bd17db4..4fa6009a5 100644--- a/scrapy/spiders/feed.py+++ b/scrapy/spiders/feed.py@@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):for result_item in self.process_results(response, ret):yield result_item- def _parse(self, response):+ def _parse(self, response, **kwargs):if not hasattr(self, 'parse_node'):raise NotConfigured('You must define parse_node method in order to scrape this XML feed')@@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):for result_item in self.process_results(response, ret):yield result_item- def _parse(self, response):+ def _parse(self, response, **kwargs):if not hasattr(self, 'parse_row'):raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')response = self.adapt_response(response)