Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Enable huge_tree for xmliter_lxml

Adrián Chavesd50f436a73ef
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9c53ab524..1c51c0c6a 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -86,7 +86,10 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]:
reader: "SupportsReadClose[bytes]" = _StreamReader(data)
node_prefix, element_name = element_name.split(":", maxsplit=1)
ns_iterator = etree.iterparse(
- reader, encoding=reader.encoding, events=("start-ns",)
+ reader,
+ encoding=reader.encoding,
+ events=("start-ns",),
+ huge_tree=True,
)
for event, (_prefix, _namespace) in ns_iterator:
if _prefix != node_prefix:
@@ -107,7 +110,10 @@ def xmliter_lxml(
reader: "SupportsReadClose[bytes]" = _StreamReader(obj)
tag = f"{{{namespace}}}{nodename}" if namespace else nodename
iterable = etree.iterparse(
- cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding
+ reader,
+ tag=tag,
+ encoding=reader.encoding,
+ huge_tree=True,
)
selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
for _, node in iterable: