Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

add tests for sitemapindex

Victor Torres5e7ecf9dc195
diff --git a/tests/test_spider.py b/tests/test_spider.py
index d5d10c9ea..8b56cfec1 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -407,6 +407,37 @@ Sitemap: /sitemap-relative-url.xml
self.assertEqual([req.url for req in spider._parse_sitemap(r)],
['http://www.example.com/english/'])
+ def test_sitemapindex_filter(self):
+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>http://www.example.com/sitemap1.xml</loc>
+ <lastmod>2004-01-01T20:00:00+00:00</lastmod>
+ </sitemap>
+ <sitemap>
+ <loc>http://www.example.com/sitemap2.xml</loc>
+ <lastmod>2005-01-01</lastmod>
+ </sitemap>
+ </sitemapindex>"""
+
+ class FilteredSitemapSpider(self.spider_class):
+ def sitemap_filter(self, entries):
+ from datetime import datetime
+ for entry in entries:
+ date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d')
+ if date_time.year > 2004:
+ yield entry
+
+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+ spider = self.spider_class("example.com")
+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+ ['http://www.example.com/sitemap1.xml',
+ 'http://www.example.com/sitemap2.xml'])
+
+ spider = FilteredSitemapSpider("example.com")
+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],
+ ['http://www.example.com/sitemap2.xml'])
+
class DeprecationTest(unittest.TestCase):