Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): skip non-element nodes in sitemap XML instead of crashing on comments (#450)

Александр М.4702b5aa9e60
diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.py
index e35918a..dcd947f 100644
--- a/scrapling/spiders/templates/sitemap.py
+++ b/scrapling/spiders/templates/sitemap.py
@@ -103,6 +103,8 @@ class SitemapSpider(Spider):
@staticmethod
def _get_type(el: Any) -> str:
+ if not isinstance(el.tag, str):
+ return ""
return etree.QName(el.tag).localname
def _sm_body(self, body: bytes, content_type: Optional[str] = None) -> SitemapResult:
diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.py
index 17c760d..362dd31 100644
--- a/tests/spiders/test_sitemap.py
+++ b/tests/spiders/test_sitemap.py
@@ -42,6 +42,16 @@ URLSET_WITH_ALTERNATES = b"""<?xml version="1.0" encoding="UTF-8"?>
</urlset>
"""
+URLSET_WITH_COMMENTS = b"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <!-- generated by example.com -->
+ <url>
+ <loc>https://example.com/posts/1</loc>
+ <!-- pending review -->
+ </url>
+</urlset>
+"""
+
INDEX_XML = b"""<?xml version="1.0" encoding="UTF-8"?>
<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<sitemap><loc>https://example.com/posts-sitemap.xml</loc></sitemap>
@@ -101,6 +111,16 @@ class TestSitemapSpiderFlow:
assert len(out) == 3
assert all(r.callback is None for r in out)
+ @pytest.mark.asyncio
+ async def test_comments_in_sitemap_are_skipped(self):
+ class S(SitemapSpider):
+ name = "s"
+ sitemap_urls = ["https://example.com/sitemap.xml"]
+
+ spider = S()
+ out = await _collect(spider._parse_sitemap(_make_response(URLSET_WITH_COMMENTS)))
+ assert [r.url for r in out] == ["https://example.com/posts/1"]
+
@pytest.mark.asyncio
async def test_sitemapindex_descends_into_children(self):
class S(SitemapSpider):