Scrapy, a fast high-level web crawling & scraping framework for Python.
add tests for the sitemap_filter method in the SitemapSpider class
Victor Torresd7d5917ff12e
diff --git a/tests/test_spider.py b/tests/test_spider.pyindex f26da2334..871852ab2 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -375,6 +375,38 @@ Sitemap: /sitemap-relative-url.xml'http://www.example.com/schweiz-deutsch/','http://www.example.com/italiano/'])+ def test_sitemap_filter(self):+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"+ xmlns:xhtml="http://www.w3.org/1999/xhtml">+ <url>+ <loc>http://www.example.com/english/</loc>+ <lastmod>2010-01-01</lastmod>+ </url>+ <url>+ <loc>http://www.example.com/portuguese/</loc>+ <lastmod>2005-01-01</lastmod>+ </url>+ </urlset>"""++ class FilteredSitemapSpider(self.spider_class):+ def sitemap_filter(self, urls):+ from datetime import datetime+ for url in urls:+ date_time = datetime.strptime(url['lastmod'], '%Y-%m-%d')+ if date_time.year > 2008:+ yield url++ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)+ spider = self.spider_class("example.com")+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],+ ['http://www.example.com/english/',+ 'http://www.example.com/portuguese/'])++ spider = FilteredSitemapSpider("example.com")+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],+ ['http://www.example.com/english/'])+class DeprecationTest(unittest.TestCase):