🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(xml template): Be specific about the node type hint
Karim shoair6e41f335f99e
diff --git a/scrapling/spiders/templates/feed.py b/scrapling/spiders/templates/feed.pyindex ea25cae..ee0e939 100644--- a/scrapling/spiders/templates/feed.py+++ b/scrapling/spiders/templates/feed.py@@ -5,6 +5,7 @@ from csv import DictReaderfrom io import StringIOfrom lxml import etree+from lxml.etree import _Elementfrom scrapling.core._types import (TYPE_CHECKING,@@ -64,7 +65,7 @@ class XMLFeedSpider(Spider):yield resultasync def parse_node(- self, response: "Response", node: Any+ self, response: "Response", node: _Element) -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:"""Override to process one feed node; `node` is a namespace-stripped `lxml` element."""raise NotImplementedError(f"{self.__class__.__name__} must implement parse_node() method")@@ -80,7 +81,7 @@ class XMLFeedSpider(Spider):raise ValueError(f"`itertag` prefix {prefix!r} is not defined in `namespaces`")return uri, name- def _iter_nodes(self, root: Any) -> Iterator[Any]:+ def _iter_nodes(self, root: _Element) -> Iterator[_Element]:uri, name = self._wanted_tag()for el in root.iter():if isinstance(el.tag, str):@@ -89,14 +90,14 @@ class XMLFeedSpider(Spider):yield self._strip_namespaces(el)@staticmethod- def _strip_namespaces(node: Any) -> Any:+ def _strip_namespaces(node: _Element) -> _Element:"""Return a copy of `node` with namespaces removed from every tag and attribute."""node = deepcopy(node)for el in node.iter():if isinstance(el.tag, str):el.tag = etree.QName(el.tag).localnamefor key in list(el.attrib):- if key.startswith("{"):+ if isinstance(key, str) and key.startswith("{"):el.attrib[etree.QName(key).localname] = el.attrib.pop(key)etree.cleanup_namespaces(node)return node