Scrapy, a fast high-level web crawling & scraping framework for Python.
add sitemap_filter attribute to SitemapSpider class
Victor Torresa5e1b7bb4724
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rstindex a08dc30f2..b0b9e0483 100644--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@ -680,6 +680,32 @@ SitemapSpiderDefault is ``sitemap_alternate_links`` disabled.+ .. attribute:: sitemap_filter++ Specifies a function to filter sitemap entries and their attributes.++ For example::++ <url>+ <loc>http://example.com/</loc>+ <lastmod>2005-01-01</lastmod>+ </url>++ We can define a ``sitemap_filter`` function to filter ``urls`` by date::++ def sitemap_filter(urls):+ from datetime import datetime+ for url in urls:+ date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')+ if date_time.year >= 2005:+ yield url++ This would retrieve only ``urls`` modified on 2005 and the following+ years.++ If you omit this attribute, all urls found in sitemaps will be+ processed, observing other attributes and their settings.+SitemapSpider examples~~~~~~~~~~~~~~~~~~~~~~diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 0ee8ba5e7..907aba243 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -17,6 +17,7 @@ class SitemapSpider(Spider):sitemap_rules = [('', 'parse')]sitemap_follow = ['']sitemap_alternate_links = False+ sitemap_filter = Nonedef __init__(self, *a, **kw):super(SitemapSpider, self).__init__(*a, **kw)@@ -43,12 +44,17 @@ class SitemapSpider(Spider):returns = Sitemap(body)+ if callable(self.sitemap_filter):+ it = self.sitemap_filter(s)+ else:+ it = s+if s.type == 'sitemapindex':- for loc in iterloc(s, self.sitemap_alternate_links):+ for loc in iterloc(it, self.sitemap_alternate_links):if any(x.search(loc) for x in self._follow):yield Request(loc, callback=self._parse_sitemap)elif s.type == 'urlset':- for loc in iterloc(s, self.sitemap_alternate_links):+ for loc in iterloc(it, self.sitemap_alternate_links):for r, c in self._cbs:if r.search(loc):yield Request(loc, callback=c)