Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix _get_tag_name() crash for non-string elem.tag (#7686) (#7687)

Gaurav Yadavdeb7e2861e61
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 1520a4ff0..03b7bf3b1 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -97,10 +97,11 @@ class Sitemap:
@staticmethod
def _get_tag_name(elem: lxml.etree._Element) -> str:
- if TYPE_CHECKING:
- assert isinstance(elem.tag, str)
- _, _, localname = elem.tag.partition("}")
- return localname or elem.tag
+ tag = elem.tag
+ if not isinstance(tag, str):
+ return ""
+ _, _, localname = tag.partition("}")
+ return localname or tag
def sitemap_urls_from_robots(
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 3599c4824..ac57e1739 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -311,3 +311,14 @@ def test_xml_entity_expansion():
"""
)
assert list(s) == [{"loc": "http://127.0.0.1:8000/"}]
+
+
+def test_sitemap_non_string_tag():
+ """With recover=True and resolve_entities=False, libxml2 >= 2.14.6 (used
+ by lxml >= 6.1.1) preserves undeclared entity reference nodes whose
+ .tag is a non-string ``Cython function`` object instead of a ``str``.
+ _get_tag_name must handle this gracefully instead of raising
+ AttributeError.
+ """
+ results = list(Sitemap(b"<url>&k;"))
+ assert results == []