Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix SitemapSpider to extract sitemap urls from robots.txt properly
orangain25c56159b862
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex eede467a8..89d96c330 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -32,7 +32,7 @@ class SitemapSpider(Spider):def _parse_sitemap(self, response):if response.url.endswith('/robots.txt'):- for url in sitemap_urls_from_robots(response.body):+ for url in sitemap_urls_from_robots(response.text):yield Request(url, callback=self._parse_sitemap)else:body = self._get_sitemap_body(response)diff --git a/tests/test_spider.py b/tests/test_spider.pyindex 4d5d4b07e..1d22c1212 100644--- a/tests/test_spider.py+++ b/tests/test_spider.py@@ -328,6 +328,18 @@ class SitemapSpiderTest(SpiderTest):r = Response(url="http://www.example.com/sitemap.xml.gz", body=self.GZBODY)self.assertSitemapBody(r, self.BODY)+ def test_get_sitemap_urls_from_robotstxt(self):+ robots = b"""# Sitemap files+Sitemap: http://example.com/sitemap.xml+Sitemap: http://example.com/sitemap-product-index.xml+"""++ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)+ spider = self.spider_class("example.com")+ self.assertEqual([req.url for req in spider._parse_sitemap(r)],+ ['http://example.com/sitemap.xml',+ 'http://example.com/sitemap-product-index.xml'])+class BaseSpiderDeprecationTest(unittest.TestCase):