Scrapy, a fast high-level web crawling & scraping framework for Python.
rename param from urls to entries
Victor Torres657f0663b3cb
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex 127c8d03e..918f1cc36 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -680,7 +680,7 @@ SitemapSpiderDefault is ``sitemap_alternate_links`` disabled.- .. method:: sitemap_filter(urls)+ .. method:: sitemap_filter(entries)Specifies a function to filter sitemap entries and their attributes.@@ -691,19 +691,19 @@ SitemapSpider<lastmod>2005-01-01</lastmod></url>- We can define a ``sitemap_filter`` function to filter ``urls`` by date::+ We can define a ``sitemap_filter`` function to filter ``entries`` by date::- def sitemap_filter(urls):+ def sitemap_filter(entries):from datetime import datetime- for url in urls:- date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')+ for entry in entries:+ date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')if date_time.year >= 2005:- yield url+ yield entry- This would retrieve only ``urls`` modified on 2005 and the following+ This would retrieve only ``entries`` modified on 2005 and the followingyears.- If you omit this method, all urls found in sitemaps will be+ If you omit this method, all entries found in sitemaps will beprocessed, observing other attributes and their settings.diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex c86e986db..534c45c70 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -31,13 +31,13 @@ class SitemapSpider(Spider):for url in self.sitemap_urls:yield Request(url, self._parse_sitemap)- def sitemap_filter(self, urls):+ def sitemap_filter(self, entries):"""This method can be used to filter sitemap entries by theirattributes, for example, you can filter locs with lastmod greaterthan a given date (see docs)."""- for url in urls:- yield url+ for entry in entries:+ yield entrydef _parse_sitemap(self, response):if response.url.endswith('/robots.txt'):diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 871852ab2..d5d10c9ea 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml</urlset>"""class FilteredSitemapSpider(self.spider_class):- def sitemap_filter(self, urls):+ def sitemap_filter(self, entries):from datetime import datetime- for url in urls:- date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')+ for entry in entries:+ date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')if date_time.year > 2008:- yield url+ yield entryr = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)spider = self.spider_class("example.com")