Scrapy, a fast high-level web crawling & scraping framework for Python.
add tests for sitemapindex
Victor Torres5e7ecf9dc195
diff --git a/tests/test_spider.py b/tests/test_spider.pyindex d5d10c9ea..8b56cfec1 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -407,6 +407,37 @@ Sitemap: /sitemap-relative-url.xmlself.assertEqual([req.url for req in spider._parse_sitemap(r)],['http://www.example.com/english/'])+ def test_sitemapindex_filter(self):+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>http://www.example.com/sitemap1.xml</loc>+ <lastmod>2004-01-01T20:00:00+00:00</lastmod>+ </sitemap>+ <sitemap>+ <loc>http://www.example.com/sitemap2.xml</loc>+ <lastmod>2005-01-01</lastmod>+ </sitemap>+ </sitemapindex>"""++ class FilteredSitemapSpider(self.spider_class):+ def sitemap_filter(self, entries):+ from datetime import datetime+ for entry in entries:+ date_time = datetime.strptime(entry['lastmod'].split('T')[0], '%Y-%m-%d')+ if date_time.year > 2004:+ yield entry++ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)+ spider = self.spider_class("example.com")+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],+ ['http://www.example.com/sitemap1.xml',+ 'http://www.example.com/sitemap2.xml'])++ spider = FilteredSitemapSpider("example.com")+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],+ ['http://www.example.com/sitemap2.xml'])+class DeprecationTest(unittest.TestCase):