🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): skip non-element nodes in sitemap XML instead of crashing on comments (#450)
Александр М.4702b5aa9e60
diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.pyindex e35918a..dcd947f 100644--- a/scrapling/spiders/templates/sitemap.py+++ b/scrapling/spiders/templates/sitemap.py@@ -103,6 +103,8 @@ class SitemapSpider(Spider):@staticmethoddef _get_type(el: Any) -> str:+ if not isinstance(el.tag, str):+ return ""return etree.QName(el.tag).localnamedef _sm_body(self, body: bytes, content_type: Optional[str] = None) -> SitemapResult:diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.pyindex 17c760d..362dd31 100644--- a/tests/spiders/test_sitemap.py+++ b/tests/spiders/test_sitemap.py@@ -42,6 +42,16 @@ URLSET_WITH_ALTERNATES = b"""<?xml version="1.0" encoding="UTF-8"?></urlset>"""+URLSET_WITH_COMMENTS = b"""<?xml version="1.0" encoding="UTF-8"?>+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <!-- generated by example.com -->+ <url>+ <loc>https://example.com/posts/1</loc>+ <!-- pending review -->+ </url>+</urlset>+"""+INDEX_XML = b"""<?xml version="1.0" encoding="UTF-8"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>https://example.com/posts-sitemap.xml</loc></sitemap>@@ -101,6 +111,16 @@ class TestSitemapSpiderFlow:assert len(out) == 3assert all(r.callback is None for r in out)+ @pytest.mark.asyncio+ async def test_comments_in_sitemap_are_skipped(self):+ class S(SitemapSpider):+ name = "s"+ sitemap_urls = ["https://example.com/sitemap.xml"]++ spider = S()+ out = await _collect(spider._parse_sitemap(_make_response(URLSET_WITH_COMMENTS)))+ assert [r.url for r in out] == ["https://example.com/posts/1"]+@pytest.mark.asyncioasync def test_sitemapindex_descends_into_children(self):class S(SitemapSpider):