Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

rename param from urls to entries

Victor Torres657f0663b3cb
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 127c8d03e..918f1cc36 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -680,7 +680,7 @@ SitemapSpider
Default is ``sitemap_alternate_links`` disabled.
- .. method:: sitemap_filter(urls)
+ .. method:: sitemap_filter(entries)
Specifies a function to filter sitemap entries and their attributes.
@@ -691,19 +691,19 @@ SitemapSpider
<lastmod>2005-01-01</lastmod>
</url>
- We can define a ``sitemap_filter`` function to filter ``urls`` by date::
+ We can define a ``sitemap_filter`` function to filter ``entries`` by date::
- def sitemap_filter(urls):
+ def sitemap_filter(entries):
from datetime import datetime
- for url in urls:
- date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+ for entry in entries:
+ date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
if date_time.year >= 2005:
- yield url
+ yield entry
- This would retrieve only ``urls`` modified on 2005 and the following
+ This would retrieve only ``entries`` modified on 2005 and the following
years.
- If you omit this method, all urls found in sitemaps will be
+ If you omit this method, all entries found in sitemaps will be
processed, observing other attributes and their settings.
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index c86e986db..534c45c70 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -31,13 +31,13 @@ class SitemapSpider(Spider):
for url in self.sitemap_urls:
yield Request(url, self._parse_sitemap)
- def sitemap_filter(self, urls):
+ def sitemap_filter(self, entries):
"""This method can be used to filter sitemap entries by their
attributes, for example, you can filter locs with lastmod greater
than a given date (see docs).
"""
- for url in urls:
- yield url
+ for entry in entries:
+ yield entry
def _parse_sitemap(self, response):
if response.url.endswith('/robots.txt'):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 871852ab2..d5d10c9ea 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -390,12 +390,12 @@ Sitemap: /sitemap-relative-url.xml
</urlset>"""
class FilteredSitemapSpider(self.spider_class):
- def sitemap_filter(self, urls):
+ def sitemap_filter(self, entries):
from datetime import datetime
- for url in urls:
- date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+ for entry in entries:
+ date_time = datetime.strptime(entry['lastmod'], '%Y-%m-%d')
if date_time.year > 2008:
- yield url
+ yield entry
r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
spider = self.spider_class("example.com")