Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

add sitemap_filter attribute to SitemapSpider class

Victor Torresa5e1b7bb4724
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index a08dc30f2..b0b9e0483 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -680,6 +680,32 @@ SitemapSpider
Default is ``sitemap_alternate_links`` disabled.
+ .. attribute:: sitemap_filter
+
+ Specifies a function to filter sitemap entries and their attributes.
+
+ For example::
+
+ <url>
+ <loc>http://example.com/</loc>
+ <lastmod>2005-01-01</lastmod>
+ </url>
+
+ We can define a ``sitemap_filter`` function to filter ``urls`` by date::
+
+ def sitemap_filter(urls):
+ from datetime import datetime
+ for url in urls:
+ date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')
+ if date_time.year >= 2005:
+ yield url
+
+ This would retrieve only ``urls`` modified on 2005 and the following
+ years.
+
+ If you omit this attribute, all urls found in sitemaps will be
+ processed, observing other attributes and their settings.
+
SitemapSpider examples
~~~~~~~~~~~~~~~~~~~~~~
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 0ee8ba5e7..907aba243 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -17,6 +17,7 @@ class SitemapSpider(Spider):
sitemap_rules = [('', 'parse')]
sitemap_follow = ['']
sitemap_alternate_links = False
+ sitemap_filter = None
def __init__(self, *a, **kw):
super(SitemapSpider, self).__init__(*a, **kw)
@@ -43,12 +44,17 @@ class SitemapSpider(Spider):
return
s = Sitemap(body)
+ if callable(self.sitemap_filter):
+ it = self.sitemap_filter(s)
+ else:
+ it = s
+
if s.type == 'sitemapindex':
- for loc in iterloc(s, self.sitemap_alternate_links):
+ for loc in iterloc(it, self.sitemap_alternate_links):
if any(x.search(loc) for x in self._follow):
yield Request(loc, callback=self._parse_sitemap)
elif s.type == 'urlset':
- for loc in iterloc(s, self.sitemap_alternate_links):
+ for loc in iterloc(it, self.sitemap_alternate_links):
for r, c in self._cbs:
if r.search(loc):
yield Request(loc, callback=c)